Merge pull request #2044 from freddiekrugerrand/1782-limitchannelhints
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Thu, 27 Apr 2023 22:02:16 +0000 (22:02 +0000)
committerGitHub <noreply@github.com>
Thu, 27 Apr 2023 22:02:16 +0000 (22:02 +0000)
Limit route hints to three channels

88 files changed:
.github/workflows/build.yml
CHANGELOG.md
README.md
ci/ci-tests.sh
fuzz/src/chanmon_consistency.rs
fuzz/src/full_stack.rs
fuzz/src/invoice_request_deser.rs
fuzz/src/refund_deser.rs
fuzz/src/router.rs
lightning-background-processor/Cargo.toml
lightning-background-processor/src/lib.rs
lightning-block-sync/Cargo.toml
lightning-custom-message/Cargo.toml
lightning-invoice/Cargo.toml
lightning-invoice/src/de.rs
lightning-invoice/src/lib.rs
lightning-invoice/src/payment.rs
lightning-invoice/src/ser.rs
lightning-invoice/src/utils.rs
lightning-invoice/tests/ser_de.rs
lightning-net-tokio/Cargo.toml
lightning-net-tokio/src/lib.rs
lightning-persister/Cargo.toml
lightning-rapid-gossip-sync/Cargo.toml
lightning-rapid-gossip-sync/src/lib.rs
lightning-rapid-gossip-sync/src/processing.rs
lightning-transaction-sync/Cargo.toml
lightning/Cargo.toml
lightning/src/blinded_path/mod.rs [new file with mode: 0644]
lightning/src/blinded_path/utils.rs [new file with mode: 0644]
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/onchaintx.rs
lightning/src/chain/package.rs
lightning/src/events/mod.rs
lightning/src/lib.rs
lightning/src/ln/chan_utils.rs
lightning/src/ln/chanmon_update_fail_tests.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/features.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/inbound_payment.rs
lightning/src/ln/monitor_tests.rs
lightning/src/ln/msgs.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/outbound_payment.rs
lightning/src/ln/payment_tests.rs
lightning/src/ln/peer_handler.rs
lightning/src/ln/priv_short_conf_tests.rs
lightning/src/ln/reload_tests.rs
lightning/src/ln/reorg_tests.rs
lightning/src/ln/shutdown_tests.rs
lightning/src/offers/invoice.rs
lightning/src/offers/invoice_request.rs
lightning/src/offers/merkle.rs
lightning/src/offers/mod.rs
lightning/src/offers/offer.rs
lightning/src/offers/parse.rs
lightning/src/offers/payer.rs
lightning/src/offers/refund.rs
lightning/src/offers/signer.rs [new file with mode: 0644]
lightning/src/offers/test_utils.rs [new file with mode: 0644]
lightning/src/onion_message/blinded_path.rs [deleted file]
lightning/src/onion_message/functional_tests.rs
lightning/src/onion_message/messenger.rs
lightning/src/onion_message/mod.rs
lightning/src/onion_message/packet.rs
lightning/src/onion_message/utils.rs [deleted file]
lightning/src/routing/gossip.rs
lightning/src/routing/router.rs
lightning/src/routing/scoring.rs
lightning/src/routing/test_utils.rs
lightning/src/routing/utxo.rs
lightning/src/sync/debug_sync.rs
lightning/src/sync/nostd_sync.rs
lightning/src/util/config.rs
lightning/src/util/crypto.rs
lightning/src/util/macro_logger.rs
lightning/src/util/ser.rs
lightning/src/util/ser_macros.rs
lightning/src/util/test_utils.rs
lightning/src/util/wakers.rs
pending_changelog/2059.txt [deleted file]
pending_changelog/2063.txt [deleted file]

index 72220d831d3bfe60085df12b16d3c0a8e5be1de2..8e02ec9da75c803caa0dfd9b797ea6b892a13298 100644 (file)
@@ -1,6 +1,16 @@
 name: Continuous Integration Checks
 
-on: [push, pull_request]
+on:
+  push:
+    branches-ignore:
+      - master
+  pull_request:
+    branches-ignore:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
@@ -180,6 +190,15 @@ jobs:
           cargo check --no-default-features --features=futures --release
           cargo doc --release
           RUSTDOCFLAGS="--cfg=anchors" cargo doc --release
+      - name: Run cargo check for Taproot build.
+        run: |
+          cargo check --release
+          cargo check --no-default-features --features=no-std --release
+          cargo check --no-default-features --features=futures --release
+          cargo doc --release
+        env:
+          RUSTFLAGS: '--cfg=anchors --cfg=taproot'
+          RUSTDOCFLAGS: '--cfg=anchors --cfg=taproot'
 
   fuzz:
     runs-on: ubuntu-latest
index d8475c003e0b2677f756fa117adf51ab9cc28f85..2874f33db0cae3c5ada1da06cfeee31de93d9892 100644 (file)
@@ -1,3 +1,100 @@
+# 0.0.115 - Apr 24, 2023 - "Rebroadcast the Bugfixes"
+
+## API Updates
+ * The MSRV of the main LDK crates has been increased to 1.48 (#2107).
+ * Attempting to claim an un-expired payment on a channel which has closed no
+   longer fails. The expiry time of payments is exposed via
+   `PaymentClaimable::claim_deadline` (#2148).
+ * `payment_metadata` is now supported in `Invoice` deserialization, sending,
+   and receiving (via a new `RecipientOnionFields` struct) (#2139, #2127).
+ * `Event::PaymentFailed` now exposes a failure reason (#2142).
+ * BOLT12 messages now support stateless generation and validation (#1989).
+ * The `NetworkGraph` is now pruned of stale data after RGS processing (#2161).
+ * Max inbound HTLCs in-flight can be changed in the handshake config (#2138).
+ * `lightning-transaction-sync` feature `esplora-async-https` was added (#2085).
+ * A `ChannelPending` event is now emitted after the initial handshake (#2098).
+ * `PaymentForwarded::outbound_amount_forwarded_msat` was added (#2136).
+ * `ChannelManager::list_channels_by_counterparty` was added (#2079).
+ * `ChannelDetails::feerate_sat_per_1000_weight` was added (#2094).
+ * `Invoice::fallback_addresses` was added to fetch `bitcoin` types (#2023).
+ * The offer/refund description is now exposed in `Invoice{,Request}` (#2206).
+
+## Backwards Compatibility
+ * Payments sent with the legacy `*_with_route` methods on LDK 0.0.115+ will no
+   longer be retryable via the LDK 0.0.114- `retry_payment` method (#2139).
+ * `Event::PaymentPathFailed::retry` was removed and will always be `None` for
+    payments initiated on 0.0.115 which fail on an earlier version (#2063).
+ * `Route`s and `PaymentParameters` with blinded path information will not be
+   readable on prior versions of LDK. Such objects are not currently constructed
+   by LDK, but may be when processing BOLT12 data in a coming release (#2146).
+ * Providing `ChannelMonitorUpdate`s generated by LDK 0.0.115 to a
+   `ChannelMonitor` on 0.0.114 or before may panic (#2059). Note that this is
+   in general unsupported, and included here only for completeness.
+
+## Bug Fixes
+ * Fixed a case where `process_events_async` may `poll` a `Future` which has
+   already completed (#2081).
+ * Fixed deserialization of `u16` arrays. This bug may have previously corrupted
+   the historical buckets in a `ProbabilisticScorer`. Users relying on the
+   historical buckets may wish to wipe their scorer on upgrade to remove corrupt
+   data rather than waiting on it to decay (#2191).
+ * The `process_events_async` task is now `Send` and can thus be polled on a
+   multi-threaded runtime (#2199).
+ * Fixed a missing macro export causing
+   `impl_writeable_tlv_based_enum{,_upgradable}` calls to not compile (#2091).
+ * Fixed compilation of `lightning-invoice` with both `no-std` and serde (#2187)
+ * Fix an issue where the `background-processor` would not wake when a
+   `ChannelMonitorUpdate` completed asynchronously, causing delays (#2090).
+ * Fix an issue where `process_events_async` would exit immediately (#2145).
+ * `Router` calls from the `ChannelManager` now call `find_route_with_id` rather
+   than `find_route`, as was intended and described in the API (#2092).
+ * Ensure `process_events_async` always exits if any sleep future returns true,
+   not just if all sleep futures repeatedly return true (#2145).
+ * `channel_update` messages no longer set the disable bit unless the peer has
+   been disconnected for some time. This should resolve cases where channels are
+   disabled for extended periods of time (#2198).
+ * We no longer remove CLN nodes from the network graph for violating the BOLT
+   spec in some cases after failing to pay through them (#2220).
+ * Fixed a debug assertion which may panic under heavy load (#2172).
+ * `CounterpartyForceClosed::peer_msg` is now wrapped in UntrustedString (#2114)
+ * Fixed a potential deadlock in `funding_transaction_generated` (#2158).
+
+## Security
+ * Transaction re-broadcasting is now substantially more aggressive, including a
+   new regular rebroadcast feature called on a timer from the
+   `background-processor` or from `ChainMonitor::rebroadcast_pending_claims`.
+   This should substantially increase transaction confirmation reliability
+   without relying on downstream `TransactionBroadcaster` implementations for
+   rebroadcasting (#2203, #2205, #2208).
+ * Implemented the changes from BOLT PRs #1031, #1032, and #1040 which resolve a
+   privacy vulnerability which allows an intermediate node on the path to
+   discover the final destination for a payment (#2062).
+
+In total, this release features 110 files changed, 11928 insertions, 6368
+deletions in 215 commits from 21 authors, in alphabetical order:
+ * Advait
+ * Alan Cohen
+ * Alec Chen
+ * Allan Douglas R. de Oliveira
+ * Arik Sosman
+ * Elias Rohrer
+ * Evan Feenstra
+ * Jeffrey Czyz
+ * John Cantrell
+ * Lucas Soriano del Pino
+ * Marc Tyndel
+ * Matt Corallo
+ * Paul Miller
+ * Steven
+ * Steven Williamson
+ * Steven Zhao
+ * Tony Giorgio
+ * Valentine Wallace
+ * Wilmer Paulino
+ * benthecarman
+ * munjesi
+
+
 # 0.0.114 - Mar 3, 2023 - "Faster Async BOLT12 Retries"
 
 ## API Updates
index 7f95427a6ab13bfb29d1fe1614e5238fb28b1bcb..a4ab59b53833afd6e672d1d67686a233b98f67ff 100644 (file)
--- a/README.md
+++ b/README.md
@@ -94,7 +94,7 @@ In general, `rust-lightning` does not provide (but LDK has implementations of):
   wanted to run your full Lightning node on a hardware wallet, you could, by
   piping the Lightning network messages over USB/serial and then sending them in
   a TCP socket from another machine.
-* private keys - again we have "default implementations", but users can chose to
+* private keys - again we have "default implementations", but users can choose to
   provide private keys to RL/LDK in any way they wish following a simple API. We
   even support a generic API for signing transactions, allowing users to run
   RL/LDK without any private keys in memory/putting private keys only on
index b2de8bc4af3cfc1b39710d012b9bda974746125d..f948cc0d63e925f94b803f2738ed0475f01cbc2b 100755 (executable)
@@ -35,13 +35,19 @@ for DIR in lightning lightning-invoice lightning-rapid-gossip-sync; do
        cargo test --verbose --color always --no-default-features --features no-std
        # check if there is a conflict between no-std and the default std feature
        cargo test --verbose --color always --features no-std
-       # check that things still pass without grind_signatures
-       # note that outbound_commitment_test only runs in this mode, because of hardcoded signature values
-       cargo test --verbose --color always --no-default-features --features std
        # check if there is a conflict between no-std and the c_bindings cfg
        RUSTFLAGS="--cfg=c_bindings" cargo test --verbose --color always --no-default-features --features=no-std
        popd
 done
+# Note that outbound_commitment_test only runs in this mode because of hardcoded signature values
+pushd lightning
+cargo test --verbose --color always --no-default-features --features=std,_test_vectors
+popd
+# This one only works for lightning-invoice
+pushd lightning-invoice
+# check that compile with no-std and serde works in lightning-invoice
+cargo test --verbose --color always --no-default-features --features no-std --features serde
+popd
 
 echo -e "\n\nTesting no-std build on a downstream no-std crate"
 # check no-std compatibility across dependencies
@@ -81,7 +87,7 @@ fi
 
 echo -e "\n\nTest futures builds"
 pushd lightning-background-processor
-cargo test --verbose --color always --no-default-features --features futures
+cargo test --verbose --color always --features futures
 popd
 
 if [ "$RUSTC_MINOR_VERSION" -gt 55 ]; then
@@ -94,4 +100,6 @@ fi
 echo -e "\n\nTest anchors builds"
 pushd lightning
 RUSTFLAGS="$RUSTFLAGS --cfg=anchors" cargo test --verbose --color always -p lightning
+echo -e "\n\nTest Taproot builds"
+RUSTFLAGS="$RUSTFLAGS --cfg=anchors --cfg=taproot" cargo test --verbose --color always -p lightning
 popd
index f6be6cdafb147b2837caaa7b4ec9f4d616123c34..0461c75772c7a39fa5176ca37bbc5f55878f4c7d 100644 (file)
@@ -41,7 +41,7 @@ use lightning::chain::keysinterface::{KeyMaterial, InMemorySigner, Recipient, En
 use lightning::events;
 use lightning::events::MessageSendEventsProvider;
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-use lightning::ln::channelmanager::{ChainParameters, ChannelDetails, ChannelManager, PaymentSendFailure, ChannelManagerReadArgs, PaymentId};
+use lightning::ln::channelmanager::{ChainParameters, ChannelDetails, ChannelManager, PaymentSendFailure, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
 use lightning::ln::channel::FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE;
 use lightning::ln::msgs::{self, CommitmentUpdate, ChannelMessageHandler, DecodeError, UpdateAddHTLC, Init};
 use lightning::ln::script::ShutdownScript;
@@ -50,7 +50,7 @@ use lightning::util::errors::APIError;
 use lightning::util::logger::Logger;
 use lightning::util::config::UserConfig;
 use lightning::util::ser::{Readable, ReadableArgs, Writeable, Writer};
-use lightning::routing::router::{InFlightHtlcs, Route, RouteHop, RouteParameters, Router};
+use lightning::routing::router::{InFlightHtlcs, Path, Route, RouteHop, RouteParameters, Router};
 
 use crate::utils::test_logger::{self, Output};
 use crate::utils::test_persister::TestPersister;
@@ -240,6 +240,7 @@ impl SignerProvider for KeyProvider {
                        [id, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, self.node_secret[31]],
                        channel_value_satoshis,
                        channel_keys_id,
+                       channel_keys_id,
                );
                let revoked_commitment = self.make_enforcement_state_cell(keys.commitment_seed);
                EnforcingSigner::new_with_revoked(keys, revoked_commitment, false)
@@ -248,7 +249,7 @@ impl SignerProvider for KeyProvider {
        fn read_chan_signer(&self, buffer: &[u8]) -> Result<Self::Signer, DecodeError> {
                let mut reader = std::io::Cursor::new(buffer);
 
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = self.make_enforcement_state_cell(inner.commitment_seed);
 
                Ok(EnforcingSigner {
@@ -351,17 +352,17 @@ fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, p
        let mut payment_id = [0; 32];
        payment_id[0..8].copy_from_slice(&payment_idx.to_ne_bytes());
        *payment_idx += 1;
-       if let Err(err) = source.send_payment(&Route {
-               paths: vec![vec![RouteHop {
+       if let Err(err) = source.send_payment_with_route(&Route {
+               paths: vec![Path { hops: vec![RouteHop {
                        pubkey: dest.get_our_node_id(),
                        node_features: dest.node_features(),
                        short_channel_id: dest_chan_id,
                        channel_features: dest.channel_features(),
                        fee_msat: amt,
                        cltv_expiry_delta: 200,
-               }]],
+               }], blinded_tail: None }],
                payment_params: None,
-       }, payment_hash, &Some(payment_secret), PaymentId(payment_id)) {
+       }, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
                check_payment_err(err);
                false
        } else { true }
@@ -373,8 +374,8 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des
        let mut payment_id = [0; 32];
        payment_id[0..8].copy_from_slice(&payment_idx.to_ne_bytes());
        *payment_idx += 1;
-       if let Err(err) = source.send_payment(&Route {
-               paths: vec![vec![RouteHop {
+       if let Err(err) = source.send_payment_with_route(&Route {
+               paths: vec![Path { hops: vec![RouteHop {
                        pubkey: middle.get_our_node_id(),
                        node_features: middle.node_features(),
                        short_channel_id: middle_chan_id,
@@ -388,9 +389,9 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des
                        channel_features: dest.channel_features(),
                        fee_msat: amt,
                        cltv_expiry_delta: 200,
-               }]],
+               }], blinded_tail: None }],
                payment_params: None,
-       }, payment_hash, &Some(payment_secret), PaymentId(payment_id)) {
+       }, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_id)) {
                check_payment_err(err);
                false
        } else { true }
@@ -526,7 +527,18 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
                                        msg.clone()
                                } else { panic!("Wrong event type"); }
                        };
+                       let events = $dest.get_and_clear_pending_events();
+                       assert_eq!(events.len(), 1);
+                       if let events::Event::ChannelPending{ ref counterparty_node_id, .. } = events[0] {
+                               assert_eq!(counterparty_node_id, &$source.get_our_node_id());
+                       } else { panic!("Wrong event type"); }
+
                        $source.handle_funding_signed(&$dest.get_our_node_id(), &funding_signed);
+                       let events = $source.get_and_clear_pending_events();
+                       assert_eq!(events.len(), 1);
+                       if let events::Event::ChannelPending{ ref counterparty_node_id, .. } = events[0] {
+                               assert_eq!(counterparty_node_id, &$dest.get_our_node_id());
+                       } else { panic!("Wrong event type"); }
 
                        funding_output
                } }
index 8ef0509a94509673af67c21b318de0431325a9c2..876a412da5fef668c74dd5b215999ece3d2901d1 100644 (file)
@@ -37,19 +37,18 @@ use lightning::chain::transaction::OutPoint;
 use lightning::chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial, EntropySource, NodeSigner, SignerProvider};
 use lightning::events::Event;
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-use lightning::ln::channelmanager::{ChainParameters, ChannelDetails, ChannelManager, PaymentId};
+use lightning::ln::channelmanager::{ChainParameters, ChannelDetails, ChannelManager, PaymentId, RecipientOnionFields, Retry};
 use lightning::ln::peer_handler::{MessageHandler,PeerManager,SocketDescriptor,IgnoringMessageHandler};
 use lightning::ln::msgs::{self, DecodeError};
 use lightning::ln::script::ShutdownScript;
 use lightning::routing::gossip::{P2PGossipSync, NetworkGraph};
 use lightning::routing::utxo::UtxoLookup;
-use lightning::routing::router::{find_route, InFlightHtlcs, PaymentParameters, Route, RouteParameters, Router};
-use lightning::routing::scoring::FixedPenaltyScorer;
+use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteParameters, Router};
 use lightning::util::config::UserConfig;
 use lightning::util::errors::APIError;
 use lightning::util::enforcing_trait_impls::{EnforcingSigner, EnforcementState};
 use lightning::util::logger::Logger;
-use lightning::util::ser::{Readable, Writeable};
+use lightning::util::ser::{Readable, ReadableArgs, Writeable};
 
 use crate::utils::test_logger;
 use crate::utils::test_persister::TestPersister;
@@ -348,6 +347,7 @@ impl SignerProvider for KeyProvider {
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, ctr],
                                channel_value_satoshis,
                                channel_keys_id,
+                               channel_keys_id,
                        )
                } else {
                        InMemorySigner::new(
@@ -360,12 +360,13 @@ impl SignerProvider for KeyProvider {
                                [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, ctr],
                                channel_value_satoshis,
                                channel_keys_id,
+                               channel_keys_id,
                        )
                }, state, false)
        }
 
        fn read_chan_signer(&self, mut data: &[u8]) -> Result<EnforcingSigner, DecodeError> {
-               let inner: InMemorySigner = Readable::read(&mut data)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut data, self)?;
                let state = Arc::new(Mutex::new(EnforcementState::new()));
 
                Ok(EnforcingSigner::new_with_revoked(
@@ -449,10 +450,8 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
        // keys subsequently generated in this test. Rather than regenerating all the messages manually,
        // it's easier to just increment the counter here so the keys don't change.
        keys_manager.counter.fetch_sub(3, Ordering::AcqRel);
-       let our_id = &keys_manager.get_node_id(Recipient::Node).unwrap();
        let network_graph = Arc::new(NetworkGraph::new(network, Arc::clone(&logger)));
        let gossip_sync = Arc::new(P2PGossipSync::new(Arc::clone(&network_graph), None, Arc::clone(&logger)));
-       let scorer = FixedPenaltyScorer::with_penalty(0);
 
        let peers = RefCell::new([false; 256]);
        let mut loss_detector = MoneyLossDetector::new(&peers, channelmanager.clone(), monitor.clone(), PeerManager::new(MessageHandler {
@@ -514,18 +513,16 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                        payment_params,
                                        final_value_msat,
                                };
-                               let random_seed_bytes: [u8; 32] = keys_manager.get_secure_random_bytes();
-                               let route = match find_route(&our_id, &params, &network_graph, None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
-                                       Ok(route) => route,
-                                       Err(_) => return,
-                               };
                                let mut payment_hash = PaymentHash([0; 32]);
                                payment_hash.0[0..8].copy_from_slice(&be64_to_array(payments_sent));
                                let mut sha = Sha256::engine();
                                sha.input(&payment_hash.0[..]);
                                payment_hash.0 = Sha256::from_engine(sha).into_inner();
                                payments_sent += 1;
-                               match channelmanager.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)) {
+                               match channelmanager.send_payment(payment_hash,
+                                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), params,
+                                       Retry::Attempts(0))
+                               {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
@@ -537,12 +534,6 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                        payment_params,
                                        final_value_msat,
                                };
-                               let random_seed_bytes: [u8; 32] = keys_manager.get_secure_random_bytes();
-                               let mut route = match find_route(&our_id, &params, &network_graph, None, Arc::clone(&logger), &scorer, &random_seed_bytes) {
-                                       Ok(route) => route,
-                                       Err(_) => return,
-                               };
-                               route.paths.push(route.paths[0].clone());
                                let mut payment_hash = PaymentHash([0; 32]);
                                payment_hash.0[0..8].copy_from_slice(&be64_to_array(payments_sent));
                                let mut sha = Sha256::engine();
@@ -552,7 +543,10 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                let mut payment_secret = PaymentSecret([0; 32]);
                                payment_secret.0[0..8].copy_from_slice(&be64_to_array(payments_sent));
                                payments_sent += 1;
-                               match channelmanager.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) {
+                               match channelmanager.send_payment(payment_hash,
+                                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0),
+                                       params, Retry::Attempts(0))
+                               {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
index aa3045ccb247e36d82990dc314caf15ecc5f0e5e..e885884891501514862d94230eb8958eb29f951f 100644 (file)
 use bitcoin::secp256k1::{KeyPair, Parity, PublicKey, Secp256k1, SecretKey, self};
 use crate::utils::test_logger;
 use core::convert::{Infallible, TryFrom};
+use lightning::blinded_path::BlindedPath;
 use lightning::chain::keysinterface::EntropySource;
 use lightning::ln::PaymentHash;
 use lightning::ln::features::BlindedHopFeatures;
 use lightning::offers::invoice::{BlindedPayInfo, UnsignedInvoice};
 use lightning::offers::invoice_request::InvoiceRequest;
 use lightning::offers::parse::SemanticError;
-use lightning::onion_message::BlindedPath;
 use lightning::util::ser::Writeable;
 
 #[inline]
@@ -74,8 +74,8 @@ fn build_response<'a, T: secp256k1::Signing + secp256k1::Verification>(
 ) -> Result<UnsignedInvoice<'a>, SemanticError> {
        let entropy_source = Randomness {};
        let paths = vec![
-               BlindedPath::new(&[pubkey(43), pubkey(44), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
-               BlindedPath::new(&[pubkey(45), pubkey(46), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
+               BlindedPath::new_for_message(&[pubkey(43), pubkey(44), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
+               BlindedPath::new_for_message(&[pubkey(45), pubkey(46), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
        ];
 
        let payinfo = vec![
index 9adaa3e953cb63c97152eef3ea3ce8f339968f26..d76607c03b02a53c5745a9a10e1d78792598f370 100644 (file)
 use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey, self};
 use crate::utils::test_logger;
 use core::convert::{Infallible, TryFrom};
+use lightning::blinded_path::BlindedPath;
 use lightning::chain::keysinterface::EntropySource;
 use lightning::ln::PaymentHash;
 use lightning::ln::features::BlindedHopFeatures;
 use lightning::offers::invoice::{BlindedPayInfo, UnsignedInvoice};
 use lightning::offers::parse::SemanticError;
 use lightning::offers::refund::Refund;
-use lightning::onion_message::BlindedPath;
 use lightning::util::ser::Writeable;
 
 #[inline]
@@ -63,8 +63,8 @@ fn build_response<'a, T: secp256k1::Signing + secp256k1::Verification>(
 ) -> Result<UnsignedInvoice<'a>, SemanticError> {
        let entropy_source = Randomness {};
        let paths = vec![
-               BlindedPath::new(&[pubkey(43), pubkey(44), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
-               BlindedPath::new(&[pubkey(45), pubkey(46), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
+               BlindedPath::new_for_message(&[pubkey(43), pubkey(44), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
+               BlindedPath::new_for_message(&[pubkey(45), pubkey(46), pubkey(42)], &entropy_source, secp_ctx).unwrap(),
        ];
 
        let payinfo = vec![
index 568dcdf0250057c68e1cf946b4b543f97c8933c0..fe6f1647f4d20c182558938b2ce475d6b7d1037b 100644 (file)
@@ -227,7 +227,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                        },
                        4 => {
                                let short_channel_id = slice_to_be64(get_slice!(8));
-                               net_graph.channel_failed(short_channel_id, false);
+                               net_graph.channel_failed_permanent(short_channel_id);
                        },
                        _ if node_pks.is_empty() => {},
                        _ => {
index af8bb25e55f558ed7a277712981da5c99b968581..1f6509e6910d5451531ca31804769b0a2b3a2249 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-background-processor"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Valentine Wallace <vwallace@protonmail.com>"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -21,10 +21,11 @@ default = ["std"]
 
 [dependencies]
 bitcoin = { version = "0.29.0", default-features = false }
-lightning = { version = "0.0.114", path = "../lightning", default-features = false }
-lightning-rapid-gossip-sync = { version = "0.0.114", path = "../lightning-rapid-gossip-sync", default-features = false }
+lightning = { version = "0.0.115", path = "../lightning", default-features = false }
+lightning-rapid-gossip-sync = { version = "0.0.115", path = "../lightning-rapid-gossip-sync", default-features = false }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
-lightning-invoice = { version = "0.22.0", path = "../lightning-invoice" }
-lightning-persister = { version = "0.0.114", path = "../lightning-persister" }
+tokio = { version = "1.14", features = [ "macros", "rt", "rt-multi-thread", "sync", "time" ] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }
+lightning-invoice = { version = "0.23.0", path = "../lightning-invoice" }
+lightning-persister = { version = "0.0.115", path = "../lightning-persister" }
index 884a7c2266420eb17f568f5da4cb65d513f6cae1..bc42c6eb68be4967479289b637b9722986a03d1e 100644 (file)
@@ -38,6 +38,8 @@ use lightning::routing::router::Router;
 use lightning::routing::scoring::{Score, WriteableScore};
 use lightning::util::logger::Logger;
 use lightning::util::persist::Persister;
+#[cfg(feature = "std")]
+use lightning::util::wakers::Sleeper;
 use lightning_rapid_gossip_sync::RapidGossipSync;
 
 use core::ops::Deref;
@@ -62,8 +64,8 @@ use alloc::vec::Vec;
 /// * Monitoring whether the [`ChannelManager`] needs to be re-persisted to disk, and if so,
 ///   writing it to disk/backups by invoking the callback given to it at startup.
 ///   [`ChannelManager`] persistence should be done in the background.
-/// * Calling [`ChannelManager::timer_tick_occurred`] and [`PeerManager::timer_tick_occurred`]
-///   at the appropriate intervals.
+/// * Calling [`ChannelManager::timer_tick_occurred`], [`ChainMonitor::rebroadcast_pending_claims`]
+///   and [`PeerManager::timer_tick_occurred`] at the appropriate intervals.
 /// * Calling [`NetworkGraph::remove_stale_channels_and_tracking`] (if a [`GossipSync`] with a
 ///   [`NetworkGraph`] is provided to [`BackgroundProcessor::start`]).
 ///
@@ -114,6 +116,18 @@ const FIRST_NETWORK_PRUNE_TIMER: u64 = 60;
 #[cfg(test)]
 const FIRST_NETWORK_PRUNE_TIMER: u64 = 1;
 
+#[cfg(not(test))]
+const REBROADCAST_TIMER: u64 = 30;
+#[cfg(test)]
+const REBROADCAST_TIMER: u64 = 1;
+
+#[cfg(feature = "futures")]
+/// core::cmp::min is not currently const, so we define a trivial (and equivalent) replacement
+const fn min_u64(a: u64, b: u64) -> u64 { if a < b { a } else { b } }
+#[cfg(feature = "futures")]
+const FASTEST_TIMER: u64 = min_u64(min_u64(FRESHNESS_TIMER, PING_TIMER),
+       min_u64(SCORER_PERSIST_TIMER, min_u64(FIRST_NETWORK_PRUNE_TIMER, REBROADCAST_TIMER)));
+
 /// Either [`P2PGossipSync`] or [`RapidGossipSync`].
 pub enum GossipSync<
        P: Deref<Target = P2PGossipSync<G, U, L>>,
@@ -227,26 +241,21 @@ fn update_scorer<'a, S: 'static + Deref<Target = SC> + Send + Sync, SC: 'a + Wri
        let mut score = scorer.lock();
        match event {
                Event::PaymentPathFailed { ref path, short_channel_id: Some(scid), .. } => {
-                       let path = path.iter().collect::<Vec<_>>();
-                       score.payment_path_failed(&path, *scid);
+                       score.payment_path_failed(path, *scid);
                },
                Event::PaymentPathFailed { ref path, payment_failed_permanently: true, .. } => {
                        // Reached if the destination explicitly failed it back. We treat this as a successful probe
                        // because the payment made it all the way to the destination with sufficient liquidity.
-                       let path = path.iter().collect::<Vec<_>>();
-                       score.probe_successful(&path);
+                       score.probe_successful(path);
                },
                Event::PaymentPathSuccessful { path, .. } => {
-                       let path = path.iter().collect::<Vec<_>>();
-                       score.payment_path_successful(&path);
+                       score.payment_path_successful(path);
                },
                Event::ProbeSuccessful { path, .. } => {
-                       let path = path.iter().collect::<Vec<_>>();
-                       score.probe_successful(&path);
+                       score.probe_successful(path);
                },
                Event::ProbeFailed { path, short_channel_id: Some(scid), .. } => {
-                       let path = path.iter().collect::<Vec<_>>();
-                       score.probe_failed(&path, *scid);
+                       score.probe_failed(path, *scid);
                },
                _ => {},
        }
@@ -256,15 +265,19 @@ macro_rules! define_run_body {
        ($persister: ident, $chain_monitor: ident, $process_chain_monitor_events: expr,
         $channel_manager: ident, $process_channel_manager_events: expr,
         $gossip_sync: ident, $peer_manager: ident, $logger: ident, $scorer: ident,
-        $loop_exit_check: expr, $await: expr, $get_timer: expr, $timer_elapsed: expr)
+        $loop_exit_check: expr, $await: expr, $get_timer: expr, $timer_elapsed: expr,
+        $check_slow_await: expr)
        => { {
                log_trace!($logger, "Calling ChannelManager's timer_tick_occurred on startup");
                $channel_manager.timer_tick_occurred();
+               log_trace!($logger, "Rebroadcasting monitor's pending claims on startup");
+               $chain_monitor.rebroadcast_pending_claims();
 
                let mut last_freshness_call = $get_timer(FRESHNESS_TIMER);
                let mut last_ping_call = $get_timer(PING_TIMER);
                let mut last_prune_call = $get_timer(FIRST_NETWORK_PRUNE_TIMER);
                let mut last_scorer_persist_call = $get_timer(SCORER_PERSIST_TIMER);
+               let mut last_rebroadcast_call = $get_timer(REBROADCAST_TIMER);
                let mut have_pruned = false;
 
                loop {
@@ -284,22 +297,30 @@ macro_rules! define_run_body {
                        // persistence.
                        $peer_manager.process_events();
 
+                       // Exit the loop if the background processor was requested to stop.
+                       if $loop_exit_check {
+                               log_trace!($logger, "Terminating background processor.");
+                               break;
+                       }
+
                        // We wait up to 100ms, but track how long it takes to detect being put to sleep,
                        // see `await_start`'s use below.
-                       let mut await_start = $get_timer(1);
+                       let mut await_start = None;
+                       if $check_slow_await { await_start = Some($get_timer(1)); }
                        let updates_available = $await;
-                       let await_slow = $timer_elapsed(&mut await_start, 1);
+                       let await_slow = if $check_slow_await { $timer_elapsed(&mut await_start.unwrap(), 1) } else { false };
 
-                       if updates_available {
-                               log_trace!($logger, "Persisting ChannelManager...");
-                               $persister.persist_manager(&*$channel_manager)?;
-                               log_trace!($logger, "Done persisting ChannelManager.");
-                       }
                        // Exit the loop if the background processor was requested to stop.
                        if $loop_exit_check {
                                log_trace!($logger, "Terminating background processor.");
                                break;
                        }
+
+                       if updates_available {
+                               log_trace!($logger, "Persisting ChannelManager...");
+                               $persister.persist_manager(&*$channel_manager)?;
+                               log_trace!($logger, "Done persisting ChannelManager.");
+                       }
                        if $timer_elapsed(&mut last_freshness_call, FRESHNESS_TIMER) {
                                log_trace!($logger, "Calling ChannelManager's timer_tick_occurred");
                                $channel_manager.timer_tick_occurred();
@@ -331,7 +352,8 @@ macro_rules! define_run_body {
                        // falling back to our usual hourly prunes. This avoids short-lived clients never
                        // pruning their network graph. We run once 60 seconds after startup before
                        // continuing our normal cadence.
-                       if $timer_elapsed(&mut last_prune_call, if have_pruned { NETWORK_PRUNE_TIMER } else { FIRST_NETWORK_PRUNE_TIMER }) {
+                       let prune_timer = if have_pruned { NETWORK_PRUNE_TIMER } else { FIRST_NETWORK_PRUNE_TIMER };
+                       if $timer_elapsed(&mut last_prune_call, prune_timer) {
                                // The network graph must not be pruned while rapid sync completion is pending
                                if let Some(network_graph) = $gossip_sync.prunable_network_graph() {
                                        #[cfg(feature = "std")] {
@@ -349,7 +371,8 @@ macro_rules! define_run_body {
 
                                        have_pruned = true;
                                }
-                               last_prune_call = $get_timer(NETWORK_PRUNE_TIMER);
+                               let prune_timer = if have_pruned { NETWORK_PRUNE_TIMER } else { FIRST_NETWORK_PRUNE_TIMER };
+                               last_prune_call = $get_timer(prune_timer);
                        }
 
                        if $timer_elapsed(&mut last_scorer_persist_call, SCORER_PERSIST_TIMER) {
@@ -361,6 +384,12 @@ macro_rules! define_run_body {
                                }
                                last_scorer_persist_call = $get_timer(SCORER_PERSIST_TIMER);
                        }
+
+                       if $timer_elapsed(&mut last_rebroadcast_call, REBROADCAST_TIMER) {
+                               log_trace!($logger, "Rebroadcasting monitor's pending claims");
+                               $chain_monitor.rebroadcast_pending_claims();
+                               last_rebroadcast_call = $get_timer(REBROADCAST_TIMER);
+                       }
                }
 
                // After we exit, ensure we persist the ChannelManager one final time - this avoids
@@ -388,15 +417,20 @@ pub(crate) mod futures_util {
        use core::task::{Poll, Waker, RawWaker, RawWakerVTable};
        use core::pin::Pin;
        use core::marker::Unpin;
-       pub(crate) struct Selector<A: Future<Output=()> + Unpin, B: Future<Output=bool> + Unpin> {
+       pub(crate) struct Selector<
+               A: Future<Output=()> + Unpin, B: Future<Output=()> + Unpin, C: Future<Output=bool> + Unpin
+       > {
                pub a: A,
                pub b: B,
+               pub c: C,
        }
        pub(crate) enum SelectorOutput {
-               A, B(bool),
+               A, B, C(bool),
        }
 
-       impl<A: Future<Output=()> + Unpin, B: Future<Output=bool> + Unpin> Future for Selector<A, B> {
+       impl<
+               A: Future<Output=()> + Unpin, B: Future<Output=()> + Unpin, C: Future<Output=bool> + Unpin
+       > Future for Selector<A, B, C> {
                type Output = SelectorOutput;
                fn poll(mut self: Pin<&mut Self>, ctx: &mut core::task::Context<'_>) -> Poll<SelectorOutput> {
                        match Pin::new(&mut self.a).poll(ctx) {
@@ -404,7 +438,11 @@ pub(crate) mod futures_util {
                                Poll::Pending => {},
                        }
                        match Pin::new(&mut self.b).poll(ctx) {
-                               Poll::Ready(res) => { return Poll::Ready(SelectorOutput::B(res)); },
+                               Poll::Ready(()) => { return Poll::Ready(SelectorOutput::B); },
+                               Poll::Pending => {},
+                       }
+                       match Pin::new(&mut self.c).poll(ctx) {
+                               Poll::Ready(res) => { return Poll::Ready(SelectorOutput::C(res)); },
                                Poll::Pending => {},
                        }
                        Poll::Pending
@@ -430,7 +468,8 @@ use core::task;
 ///
 /// `sleeper` should return a future which completes in the given amount of time and returns a
 /// boolean indicating whether the background processing should exit. Once `sleeper` returns a
-/// future which outputs true, the loop will exit and this function's future will complete.
+/// future which outputs `true`, the loop will exit and this function's future will complete.
+/// The `sleeper` future is free to return early after it has triggered the exit condition.
 ///
 /// See [`BackgroundProcessor::start`] for information on which actions this handles.
 ///
@@ -438,6 +477,92 @@ use core::task;
 /// feature, doing so will skip calling [`NetworkGraph::remove_stale_channels_and_tracking`],
 /// you should call [`NetworkGraph::remove_stale_channels_and_tracking_with_time`] regularly
 /// manually instead.
+///
+/// The `mobile_interruptable_platform` flag should be set if we're currently running on a
+/// mobile device, where we may need to check for interruption of the application regularly. If you
+/// are unsure, you should set the flag, as the performance impact of it is minimal unless there
+/// are hundreds or thousands of simultaneous process calls running.
+///
+/// For example, in order to process background events in a [Tokio](https://tokio.rs/) task, you
+/// could setup `process_events_async` like this:
+/// ```
+/// # struct MyPersister {}
+/// # impl lightning::util::persist::KVStorePersister for MyPersister {
+/// #     fn persist<W: lightning::util::ser::Writeable>(&self, key: &str, object: &W) -> lightning::io::Result<()> { Ok(()) }
+/// # }
+/// # struct MyEventHandler {}
+/// # impl MyEventHandler {
+/// #     async fn handle_event(&self, _: lightning::events::Event) {}
+/// # }
+/// # #[derive(Eq, PartialEq, Clone, Hash)]
+/// # struct MySocketDescriptor {}
+/// # impl lightning::ln::peer_handler::SocketDescriptor for MySocketDescriptor {
+/// #     fn send_data(&mut self, _data: &[u8], _resume_read: bool) -> usize { 0 }
+/// #     fn disconnect_socket(&mut self) {}
+/// # }
+/// # use std::sync::{Arc, Mutex};
+/// # use std::sync::atomic::{AtomicBool, Ordering};
+/// # use lightning_background_processor::{process_events_async, GossipSync};
+/// # type MyBroadcaster = dyn lightning::chain::chaininterface::BroadcasterInterface + Send + Sync;
+/// # type MyFeeEstimator = dyn lightning::chain::chaininterface::FeeEstimator + Send + Sync;
+/// # type MyNodeSigner = dyn lightning::chain::keysinterface::NodeSigner + Send + Sync;
+/// # type MyUtxoLookup = dyn lightning::routing::utxo::UtxoLookup + Send + Sync;
+/// # type MyFilter = dyn lightning::chain::Filter + Send + Sync;
+/// # type MyLogger = dyn lightning::util::logger::Logger + Send + Sync;
+/// # type MyChainMonitor = lightning::chain::chainmonitor::ChainMonitor<lightning::chain::keysinterface::InMemorySigner, Arc<MyFilter>, Arc<MyBroadcaster>, Arc<MyFeeEstimator>, Arc<MyLogger>, Arc<MyPersister>>;
+/// # type MyPeerManager = lightning::ln::peer_handler::SimpleArcPeerManager<MySocketDescriptor, MyChainMonitor, MyBroadcaster, MyFeeEstimator, MyUtxoLookup, MyLogger>;
+/// # type MyNetworkGraph = lightning::routing::gossip::NetworkGraph<Arc<MyLogger>>;
+/// # type MyGossipSync = lightning::routing::gossip::P2PGossipSync<Arc<MyNetworkGraph>, Arc<MyUtxoLookup>, Arc<MyLogger>>;
+/// # type MyChannelManager = lightning::ln::channelmanager::SimpleArcChannelManager<MyChainMonitor, MyBroadcaster, MyFeeEstimator, MyLogger>;
+/// # type MyScorer = Mutex<lightning::routing::scoring::ProbabilisticScorer<Arc<MyNetworkGraph>, Arc<MyLogger>>>;
+///
+/// # async fn setup_background_processing(my_persister: Arc<MyPersister>, my_event_handler: Arc<MyEventHandler>, my_chain_monitor: Arc<MyChainMonitor>, my_channel_manager: Arc<MyChannelManager>, my_gossip_sync: Arc<MyGossipSync>, my_logger: Arc<MyLogger>, my_scorer: Arc<MyScorer>, my_peer_manager: Arc<MyPeerManager>) {
+///    let background_persister = Arc::clone(&my_persister);
+///    let background_event_handler = Arc::clone(&my_event_handler);
+///    let background_chain_mon = Arc::clone(&my_chain_monitor);
+///    let background_chan_man = Arc::clone(&my_channel_manager);
+///    let background_gossip_sync = GossipSync::p2p(Arc::clone(&my_gossip_sync));
+///    let background_peer_man = Arc::clone(&my_peer_manager);
+///    let background_logger = Arc::clone(&my_logger);
+///    let background_scorer = Arc::clone(&my_scorer);
+///
+///    // Setup the sleeper.
+///    let (stop_sender, stop_receiver) = tokio::sync::watch::channel(());
+///
+///    let sleeper = move |d| {
+///            let mut receiver = stop_receiver.clone();
+///            Box::pin(async move {
+///                    tokio::select!{
+///                            _ = tokio::time::sleep(d) => false,
+///                            _ = receiver.changed() => true,
+///                    }
+///            })
+///    };
+///
+///    let mobile_interruptable_platform = false;
+///
+///    let handle = tokio::spawn(async move {
+///            process_events_async(
+///                    background_persister,
+///                    |e| background_event_handler.handle_event(e),
+///                    background_chain_mon,
+///                    background_chan_man,
+///                    background_gossip_sync,
+///                    background_peer_man,
+///                    background_logger,
+///                    Some(background_scorer),
+///                    sleeper,
+///                    mobile_interruptable_platform,
+///                    )
+///                    .await
+///                    .expect("Failed to process events");
+///    });
+///
+///    // Stop the background processing.
+///    stop_sender.send(()).unwrap();
+///    handle.await.unwrap();
+///    # }
+///```
 #[cfg(feature = "futures")]
 pub async fn process_events_async<
        'a,
@@ -473,7 +598,7 @@ pub async fn process_events_async<
 >(
        persister: PS, event_handler: EventHandler, chain_monitor: M, channel_manager: CM,
        gossip_sync: GossipSync<PGS, RGS, G, UL, L>, peer_manager: PM, logger: L, scorer: Option<S>,
-       sleeper: Sleeper,
+       sleeper: Sleeper, mobile_interruptable_platform: bool,
 ) -> Result<(), lightning::io::Error>
 where
        UL::Target: 'static + UtxoLookup,
@@ -493,7 +618,7 @@ where
        UMH::Target: 'static + CustomMessageHandler,
        PS::Target: 'static + Persister<'a, CW, T, ES, NS, SP, F, R, L, SC>,
 {
-       let mut should_break = true;
+       let mut should_break = false;
        let async_event_handler = |event| {
                let network_graph = gossip_sync.network_graph();
                let event_handler = &event_handler;
@@ -514,11 +639,13 @@ where
                gossip_sync, peer_manager, logger, scorer, should_break, {
                        let fut = Selector {
                                a: channel_manager.get_persistable_update_future(),
-                               b: sleeper(Duration::from_millis(100)),
+                               b: chain_monitor.get_update_future(),
+                               c: sleeper(if mobile_interruptable_platform { Duration::from_millis(100) } else { Duration::from_secs(FASTEST_TIMER) }),
                        };
                        match fut.await {
                                SelectorOutput::A => true,
-                               SelectorOutput::B(exit) => {
+                               SelectorOutput::B => false,
+                               SelectorOutput::C(exit) => {
                                        should_break = exit;
                                        false
                                }
@@ -527,8 +654,11 @@ where
                |fut: &mut SleepFuture, _| {
                        let mut waker = dummy_waker();
                        let mut ctx = task::Context::from_waker(&mut waker);
-                       core::pin::Pin::new(fut).poll(&mut ctx).is_ready()
-               })
+                       match core::pin::Pin::new(fut).poll(&mut ctx) {
+                               task::Poll::Ready(exit) => { should_break = exit; true },
+                               task::Poll::Pending => false,
+                       }
+               }, mobile_interruptable_platform)
 }
 
 #[cfg(feature = "std")]
@@ -643,8 +773,11 @@ impl BackgroundProcessor {
                        define_run_body!(persister, chain_monitor, chain_monitor.process_pending_events(&event_handler),
                                channel_manager, channel_manager.process_pending_events(&event_handler),
                                gossip_sync, peer_manager, logger, scorer, stop_thread.load(Ordering::Acquire),
-                               channel_manager.await_persistable_update_timeout(Duration::from_millis(100)),
-                               |_| Instant::now(), |time: &Instant, dur| time.elapsed().as_secs() > dur)
+                               Sleeper::from_two_futures(
+                                       channel_manager.get_persistable_update_future(),
+                                       chain_monitor.get_update_future()
+                               ).wait_timeout(Duration::from_millis(100)),
+                               |_| Instant::now(), |time: &Instant, dur| time.elapsed().as_secs() > dur, false)
                });
                Self { stop_thread: stop_thread_clone, thread_handle: Some(handle) }
        }
@@ -710,7 +843,7 @@ mod tests {
        use lightning::chain::keysinterface::{InMemorySigner, KeysManager};
        use lightning::chain::transaction::OutPoint;
        use lightning::events::{Event, PathFailure, MessageSendEventsProvider, MessageSendEvent};
-       use lightning::get_event_msg;
+       use lightning::{get_event_msg, get_event};
        use lightning::ln::PaymentHash;
        use lightning::ln::channelmanager;
        use lightning::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChainParameters, MIN_CLTV_EXPIRY_DELTA, PaymentId};
@@ -718,7 +851,7 @@ mod tests {
        use lightning::ln::msgs::{ChannelMessageHandler, Init};
        use lightning::ln::peer_handler::{PeerManager, MessageHandler, SocketDescriptor, IgnoringMessageHandler};
        use lightning::routing::gossip::{NetworkGraph, NodeId, P2PGossipSync};
-       use lightning::routing::router::{DefaultRouter, RouteHop};
+       use lightning::routing::router::{DefaultRouter, Path, RouteHop};
        use lightning::routing::scoring::{ChannelUsage, Score};
        use lightning::util::config::UserConfig;
        use lightning::util::ser::Writeable;
@@ -726,7 +859,7 @@ mod tests {
        use lightning::util::persist::KVStorePersister;
        use lightning_persister::FilesystemPersister;
        use std::collections::VecDeque;
-       use std::fs;
+       use std::{fs, env};
        use std::path::PathBuf;
        use std::sync::{Arc, Mutex};
        use std::sync::mpsc::SyncSender;
@@ -834,7 +967,10 @@ mod tests {
 
                        if key == "network_graph" {
                                if let Some(sender) = &self.graph_persistence_notifier {
-                                       sender.send(()).unwrap();
+                                       match sender.send(()) {
+                                               Ok(()) => {},
+                                               Err(std::sync::mpsc::SendError(())) => println!("Persister failed to notify as receiver went away."),
+                                       }
                                };
 
                                if let Some((error, message)) = self.graph_error {
@@ -858,10 +994,10 @@ mod tests {
 
        #[derive(Debug)]
        enum TestResult {
-               PaymentFailure { path: Vec<RouteHop>, short_channel_id: u64 },
-               PaymentSuccess { path: Vec<RouteHop> },
-               ProbeFailure { path: Vec<RouteHop> },
-               ProbeSuccess { path: Vec<RouteHop> },
+               PaymentFailure { path: Path, short_channel_id: u64 },
+               PaymentSuccess { path: Path },
+               ProbeFailure { path: Path },
+               ProbeSuccess { path: Path },
        }
 
        impl TestScorer {
@@ -883,11 +1019,11 @@ mod tests {
                        &self, _short_channel_id: u64, _source: &NodeId, _target: &NodeId, _usage: ChannelUsage
                ) -> u64 { unimplemented!(); }
 
-               fn payment_path_failed(&mut self, actual_path: &[&RouteHop], actual_short_channel_id: u64) {
+               fn payment_path_failed(&mut self, actual_path: &Path, actual_short_channel_id: u64) {
                        if let Some(expectations) = &mut self.event_expectations {
                                match expectations.pop_front().unwrap() {
                                        TestResult::PaymentFailure { path, short_channel_id } => {
-                                               assert_eq!(actual_path, &path.iter().collect::<Vec<_>>()[..]);
+                                               assert_eq!(actual_path, &path);
                                                assert_eq!(actual_short_channel_id, short_channel_id);
                                        },
                                        TestResult::PaymentSuccess { path } => {
@@ -903,14 +1039,14 @@ mod tests {
                        }
                }
 
-               fn payment_path_successful(&mut self, actual_path: &[&RouteHop]) {
+               fn payment_path_successful(&mut self, actual_path: &Path) {
                        if let Some(expectations) = &mut self.event_expectations {
                                match expectations.pop_front().unwrap() {
                                        TestResult::PaymentFailure { path, .. } => {
                                                panic!("Unexpected payment path failure: {:?}", path)
                                        },
                                        TestResult::PaymentSuccess { path } => {
-                                               assert_eq!(actual_path, &path.iter().collect::<Vec<_>>()[..]);
+                                               assert_eq!(actual_path, &path);
                                        },
                                        TestResult::ProbeFailure { path } => {
                                                panic!("Unexpected probe failure: {:?}", path)
@@ -922,7 +1058,7 @@ mod tests {
                        }
                }
 
-               fn probe_failed(&mut self, actual_path: &[&RouteHop], _: u64) {
+               fn probe_failed(&mut self, actual_path: &Path, _: u64) {
                        if let Some(expectations) = &mut self.event_expectations {
                                match expectations.pop_front().unwrap() {
                                        TestResult::PaymentFailure { path, .. } => {
@@ -932,7 +1068,7 @@ mod tests {
                                                panic!("Unexpected payment path success: {:?}", path)
                                        },
                                        TestResult::ProbeFailure { path } => {
-                                               assert_eq!(actual_path, &path.iter().collect::<Vec<_>>()[..]);
+                                               assert_eq!(actual_path, &path);
                                        },
                                        TestResult::ProbeSuccess { path } => {
                                                panic!("Unexpected probe success: {:?}", path)
@@ -940,7 +1076,7 @@ mod tests {
                                }
                        }
                }
-               fn probe_successful(&mut self, actual_path: &[&RouteHop]) {
+               fn probe_successful(&mut self, actual_path: &Path) {
                        if let Some(expectations) = &mut self.event_expectations {
                                match expectations.pop_front().unwrap() {
                                        TestResult::PaymentFailure { path, .. } => {
@@ -953,7 +1089,7 @@ mod tests {
                                                panic!("Unexpected probe failure: {:?}", path)
                                        },
                                        TestResult::ProbeSuccess { path } => {
-                                               assert_eq!(actual_path, &path.iter().collect::<Vec<_>>()[..]);
+                                               assert_eq!(actual_path, &path);
                                        }
                                }
                        }
@@ -980,20 +1116,22 @@ mod tests {
                path.to_str().unwrap().to_string()
        }
 
-       fn create_nodes(num_nodes: usize, persist_dir: String) -> Vec<Node> {
+       fn create_nodes(num_nodes: usize, persist_dir: &str) -> (String, Vec<Node>) {
+               let persist_temp_path = env::temp_dir().join(persist_dir);
+               let persist_dir = persist_temp_path.to_string_lossy().to_string();
+               let network = Network::Testnet;
                let mut nodes = Vec::new();
                for i in 0..num_nodes {
-                       let tx_broadcaster = Arc::new(test_utils::TestBroadcaster{txn_broadcasted: Mutex::new(Vec::new()), blocks: Arc::new(Mutex::new(Vec::new()))});
+                       let tx_broadcaster = Arc::new(test_utils::TestBroadcaster::new(network));
                        let fee_estimator = Arc::new(test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) });
                        let logger = Arc::new(test_utils::TestLogger::with_id(format!("node {}", i)));
-                       let network = Network::Testnet;
                        let genesis_block = genesis_block(network);
                        let network_graph = Arc::new(NetworkGraph::new(network, logger.clone()));
                        let scorer = Arc::new(Mutex::new(TestScorer::new()));
                        let seed = [i as u8; 32];
                        let router = Arc::new(DefaultRouter::new(network_graph.clone(), logger.clone(), seed, scorer.clone()));
                        let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet));
-                       let persister = Arc::new(FilesystemPersister::new(format!("{}_persister_{}", persist_dir, i)));
+                       let persister = Arc::new(FilesystemPersister::new(format!("{}_persister_{}", &persist_dir, i)));
                        let now = Duration::from_secs(genesis_block.header.time as u64);
                        let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_nanos()));
                        let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new(Some(chain_source.clone()), tx_broadcaster.clone(), logger.clone(), fee_estimator.clone(), persister.clone()));
@@ -1015,7 +1153,7 @@ mod tests {
                        }
                }
 
-               nodes
+               (persist_dir, nodes)
        }
 
        macro_rules! open_channel {
@@ -1024,7 +1162,11 @@ mod tests {
                        let events = $node_a.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
                        let (temporary_channel_id, tx) = handle_funding_generation_ready!(events[0], $channel_value);
-                       end_open_channel!($node_a, $node_b, temporary_channel_id, tx);
+                       $node_a.node.funding_transaction_generated(&temporary_channel_id, &$node_b.node.get_our_node_id(), tx.clone()).unwrap();
+                       $node_b.node.handle_funding_created(&$node_a.node.get_our_node_id(), &get_event_msg!($node_a, MessageSendEvent::SendFundingCreated, $node_b.node.get_our_node_id()));
+                       get_event!($node_b, Event::ChannelPending);
+                       $node_a.node.handle_funding_signed(&$node_b.node.get_our_node_id(), &get_event_msg!($node_b, MessageSendEvent::SendFundingSigned, $node_a.node.get_our_node_id()));
+                       get_event!($node_a, Event::ChannelPending);
                        tx
                }}
        }
@@ -1054,14 +1196,6 @@ mod tests {
                }}
        }
 
-       macro_rules! end_open_channel {
-               ($node_a: expr, $node_b: expr, $temporary_channel_id: expr, $tx: expr) => {{
-                       $node_a.node.funding_transaction_generated(&$temporary_channel_id, &$node_b.node.get_our_node_id(), $tx.clone()).unwrap();
-                       $node_b.node.handle_funding_created(&$node_a.node.get_our_node_id(), &get_event_msg!($node_a, MessageSendEvent::SendFundingCreated, $node_b.node.get_our_node_id()));
-                       $node_a.node.handle_funding_signed(&$node_b.node.get_our_node_id(), &get_event_msg!($node_b, MessageSendEvent::SendFundingSigned, $node_a.node.get_our_node_id()));
-               }}
-       }
-
        fn confirm_transaction_depth(node: &mut Node, tx: &Transaction, depth: u32) {
                for i in 1..=depth {
                        let prev_blockhash = node.best_block.block_hash();
@@ -1091,7 +1225,7 @@ mod tests {
                // Test that when a new channel is created, the ChannelManager needs to be re-persisted with
                // updates. Also test that when new updates are available, the manager signals that it needs
                // re-persistence and is successfully re-persisted.
-               let nodes = create_nodes(2, "test_background_processor".to_string());
+               let (persist_dir, nodes) = create_nodes(2, "test_background_processor");
 
                // Go through the channel creation process so that each node has something to persist. Since
                // open_channel consumes events, it must complete before starting BackgroundProcessor to
@@ -1129,7 +1263,7 @@ mod tests {
                }
 
                // Check that the initial channel manager data is persisted as expected.
-               let filepath = get_full_filepath("test_background_processor_persister_0".to_string(), "manager".to_string());
+               let filepath = get_full_filepath(format!("{}_persister_0", &persist_dir), "manager".to_string());
                check_persisted_data!(nodes[0].node, filepath.clone());
 
                loop {
@@ -1146,42 +1280,49 @@ mod tests {
                }
 
                // Check network graph is persisted
-               let filepath = get_full_filepath("test_background_processor_persister_0".to_string(), "network_graph".to_string());
+               let filepath = get_full_filepath(format!("{}_persister_0", &persist_dir), "network_graph".to_string());
                check_persisted_data!(nodes[0].network_graph, filepath.clone());
 
                // Check scorer is persisted
-               let filepath = get_full_filepath("test_background_processor_persister_0".to_string(), "scorer".to_string());
+               let filepath = get_full_filepath(format!("{}_persister_0", &persist_dir), "scorer".to_string());
                check_persisted_data!(nodes[0].scorer, filepath.clone());
 
-               assert!(bg_processor.stop().is_ok());
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
+               }
        }
 
        #[test]
        fn test_timer_tick_called() {
-               // Test that ChannelManager's and PeerManager's `timer_tick_occurred` is called every
-               // `FRESHNESS_TIMER`.
-               let nodes = create_nodes(1, "test_timer_tick_called".to_string());
+               // Test that `ChannelManager::timer_tick_occurred` is called every `FRESHNESS_TIMER`,
+               // `ChainMonitor::rebroadcast_pending_claims` is called every `REBROADCAST_TIMER`, and
+               // `PeerManager::timer_tick_occurred` every `PING_TIMER`.
+               let (_, nodes) = create_nodes(1, "test_timer_tick_called");
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
                let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
-                       let desired_log = "Calling ChannelManager's timer_tick_occurred".to_string();
-                       let second_desired_log = "Calling PeerManager's timer_tick_occurred".to_string();
-                       if log_entries.get(&("lightning_background_processor".to_string(), desired_log)).is_some() &&
-                                       log_entries.get(&("lightning_background_processor".to_string(), second_desired_log)).is_some() {
+                       let desired_log_1 = "Calling ChannelManager's timer_tick_occurred".to_string();
+                       let desired_log_2 = "Calling PeerManager's timer_tick_occurred".to_string();
+                       let desired_log_3 = "Rebroadcasting monitor's pending claims".to_string();
+                       if log_entries.get(&("lightning_background_processor".to_string(), desired_log_1)).is_some() &&
+                               log_entries.get(&("lightning_background_processor".to_string(), desired_log_2)).is_some() &&
+                               log_entries.get(&("lightning_background_processor".to_string(), desired_log_3)).is_some() {
                                break
                        }
                }
 
-               assert!(bg_processor.stop().is_ok());
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
+               }
        }
 
        #[test]
        fn test_channel_manager_persist_error() {
                // Test that if we encounter an error during manager persistence, the thread panics.
-               let nodes = create_nodes(2, "test_persist_error".to_string());
+               let (_, nodes) = create_nodes(2, "test_persist_error");
                open_channel!(nodes[0], nodes[1], 100000);
 
                let data_dir = nodes[0].persister.get_data_dir();
@@ -1197,10 +1338,39 @@ mod tests {
                }
        }
 
+       #[tokio::test]
+       #[cfg(feature = "futures")]
+       async fn test_channel_manager_persist_error_async() {
+               // Test that if we encounter an error during manager persistence, the thread panics.
+               let (_, nodes) = create_nodes(2, "test_persist_error_sync");
+               open_channel!(nodes[0], nodes[1], 100000);
+
+               let data_dir = nodes[0].persister.get_data_dir();
+               let persister = Arc::new(Persister::new(data_dir).with_manager_error(std::io::ErrorKind::Other, "test"));
+
+               let bp_future = super::process_events_async(
+                       persister, |_: _| {async {}}, nodes[0].chain_monitor.clone(), nodes[0].node.clone(),
+                       nodes[0].rapid_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(),
+                       Some(nodes[0].scorer.clone()), move |dur: Duration| {
+                               Box::pin(async move {
+                                       tokio::time::sleep(dur).await;
+                                       false // Never exit
+                               })
+                       }, false,
+               );
+               match bp_future.await {
+                       Ok(_) => panic!("Expected error persisting manager"),
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::Other);
+                               assert_eq!(e.get_ref().unwrap().to_string(), "test");
+                       },
+               }
+       }
+
        #[test]
        fn test_network_graph_persist_error() {
                // Test that if we encounter an error during network graph persistence, an error gets returned.
-               let nodes = create_nodes(2, "test_persist_network_graph_error".to_string());
+               let (_, nodes) = create_nodes(2, "test_persist_network_graph_error");
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_graph_error(std::io::ErrorKind::Other, "test"));
                let event_handler = |_: _| {};
@@ -1218,7 +1388,7 @@ mod tests {
        #[test]
        fn test_scorer_persist_error() {
                // Test that if we encounter an error during scorer persistence, an error gets returned.
-               let nodes = create_nodes(2, "test_persist_scorer_error".to_string());
+               let (_, nodes) = create_nodes(2, "test_persist_scorer_error");
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_scorer_error(std::io::ErrorKind::Other, "test"));
                let event_handler = |_: _| {};
@@ -1235,15 +1405,17 @@ mod tests {
 
        #[test]
        fn test_background_event_handling() {
-               let mut nodes = create_nodes(2, "test_background_event_handling".to_string());
+               let (_, mut nodes) = create_nodes(2, "test_background_event_handling");
                let channel_value = 100000;
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir.clone()));
 
                // Set up a background event handler for FundingGenerationReady events.
-               let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+               let (funding_generation_send, funding_generation_recv) = std::sync::mpsc::sync_channel(1);
+               let (channel_pending_send, channel_pending_recv) = std::sync::mpsc::sync_channel(1);
                let event_handler = move |event: Event| match event {
-                       Event::FundingGenerationReady { .. } => sender.send(handle_funding_generation_ready!(event, channel_value)).unwrap(),
+                       Event::FundingGenerationReady { .. } => funding_generation_send.send(handle_funding_generation_ready!(event, channel_value)).unwrap(),
+                       Event::ChannelPending { .. } => channel_pending_send.send(()).unwrap(),
                        Event::ChannelReady { .. } => {},
                        _ => panic!("Unexpected event: {:?}", event),
                };
@@ -1252,10 +1424,15 @@ mod tests {
 
                // Open a channel and check that the FundingGenerationReady event was handled.
                begin_open_channel!(nodes[0], nodes[1], channel_value);
-               let (temporary_channel_id, funding_tx) = receiver
+               let (temporary_channel_id, funding_tx) = funding_generation_recv
                        .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
                        .expect("FundingGenerationReady not handled within deadline");
-               end_open_channel!(nodes[0], nodes[1], temporary_channel_id, funding_tx);
+               nodes[0].node.funding_transaction_generated(&temporary_channel_id, &nodes[1].node.get_our_node_id(), funding_tx.clone()).unwrap();
+               nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
+               get_event!(nodes[1], Event::ChannelPending);
+               nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
+               let _ = channel_pending_recv.recv_timeout(Duration::from_secs(EVENT_DEADLINE))
+                       .expect("ChannelPending not handled within deadline");
 
                // Confirm the funding transaction.
                confirm_transaction(&mut nodes[0], &funding_tx);
@@ -1267,7 +1444,9 @@ mod tests {
                nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_funding);
                let _bs_channel_update = get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[0].node.get_our_node_id());
 
-               assert!(bg_processor.stop().is_ok());
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
+               }
 
                // Set up a background event handler for SpendableOutputs events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
@@ -1293,12 +1472,14 @@ mod tests {
                        _ => panic!("Unexpected event: {:?}", event),
                }
 
-               assert!(bg_processor.stop().is_ok());
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
+               }
        }
 
        #[test]
        fn test_scorer_persistence() {
-               let nodes = create_nodes(2, "test_scorer_persistence".to_string());
+               let (_, nodes) = create_nodes(2, "test_scorer_persistence");
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
                let event_handler = |_: _| {};
@@ -1312,73 +1493,214 @@ mod tests {
                        }
                }
 
-               assert!(bg_processor.stop().is_ok());
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
+               }
+       }
+
+       macro_rules! do_test_not_pruning_network_graph_until_graph_sync_completion {
+               ($nodes: expr, $receive: expr, $sleep: expr) => {
+                       let features = ChannelFeatures::empty();
+                       $nodes[0].network_graph.add_channel_from_partial_announcement(
+                               42, 53, features, $nodes[0].node.get_our_node_id(), $nodes[1].node.get_our_node_id()
+                       ).expect("Failed to update channel from partial announcement");
+                       let original_graph_description = $nodes[0].network_graph.to_string();
+                       assert!(original_graph_description.contains("42: features: 0000, node_one:"));
+                       assert_eq!($nodes[0].network_graph.read_only().channels().len(), 1);
+
+                       loop {
+                               $sleep;
+                               let log_entries = $nodes[0].logger.lines.lock().unwrap();
+                               let loop_counter = "Calling ChannelManager's timer_tick_occurred".to_string();
+                               if *log_entries.get(&("lightning_background_processor".to_string(), loop_counter))
+                                       .unwrap_or(&0) > 1
+                               {
+                                       // Wait until the loop has gone around at least twice.
+                                       break
+                               }
+                       }
+
+                       let initialization_input = vec![
+                               76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
+                               79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
+                               0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
+                               187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
+                               157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
+                               88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
+                               204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
+                               181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
+                               110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
+                               76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
+                               226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
+                               0, 0, 0, 1, 0, 0, 0, 0, 58, 85, 116, 216, 255, 8, 153, 192, 0, 2, 27, 0, 0, 25, 0, 0,
+                               0, 1, 0, 0, 0, 125, 255, 2, 68, 226, 0, 6, 11, 0, 1, 5, 0, 0, 0, 0, 29, 129, 25, 192,
+                       ];
+                       $nodes[0].rapid_gossip_sync.update_network_graph_no_std(&initialization_input[..], Some(1642291930)).unwrap();
+
+                       // this should have added two channels and pruned the previous one.
+                       assert_eq!($nodes[0].network_graph.read_only().channels().len(), 2);
+
+                       $receive.expect("Network graph not pruned within deadline");
+
+                       // all channels should now be pruned
+                       assert_eq!($nodes[0].network_graph.read_only().channels().len(), 0);
+               }
        }
 
        #[test]
        fn test_not_pruning_network_graph_until_graph_sync_completion() {
-               let nodes = create_nodes(2, "test_not_pruning_network_graph_until_graph_sync_completion".to_string());
-               let data_dir = nodes[0].persister.get_data_dir();
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+
+               let (_, nodes) = create_nodes(2, "test_not_pruning_network_graph_until_graph_sync_completion");
+               let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_graph_persistence_notifier(sender));
-               let network_graph = nodes[0].network_graph.clone();
-               let features = ChannelFeatures::empty();
-               network_graph.add_channel_from_partial_announcement(42, 53, features, nodes[0].node.get_our_node_id(), nodes[1].node.get_our_node_id())
-                       .expect("Failed to update channel from partial announcement");
-               let original_graph_description = network_graph.to_string();
-               assert!(original_graph_description.contains("42: features: 0000, node_one:"));
-               assert_eq!(network_graph.read_only().channels().len(), 1);
 
                let event_handler = |_: _| {};
                let background_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].rapid_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
-               loop {
-                       let log_entries = nodes[0].logger.lines.lock().unwrap();
-                       let loop_counter = "Calling ChannelManager's timer_tick_occurred".to_string();
-                       if *log_entries.get(&("lightning_background_processor".to_string(), loop_counter))
-                               .unwrap_or(&0) > 1
-                       {
-                               // Wait until the loop has gone around at least twice.
-                               break
+               do_test_not_pruning_network_graph_until_graph_sync_completion!(nodes,
+                       receiver.recv_timeout(Duration::from_secs(super::FIRST_NETWORK_PRUNE_TIMER * 5)),
+                       std::thread::sleep(Duration::from_millis(1)));
+
+               background_processor.stop().unwrap();
+       }
+
+       #[tokio::test]
+       #[cfg(feature = "futures")]
+       async fn test_not_pruning_network_graph_until_graph_sync_completion_async() {
+               let (sender, receiver) = std::sync::mpsc::sync_channel(1);
+
+               let (_, nodes) = create_nodes(2, "test_not_pruning_network_graph_until_graph_sync_completion_async");
+               let data_dir = nodes[0].persister.get_data_dir();
+               let persister = Arc::new(Persister::new(data_dir).with_graph_persistence_notifier(sender));
+
+               let (exit_sender, exit_receiver) = tokio::sync::watch::channel(());
+               let bp_future = super::process_events_async(
+                       persister, |_: _| {async {}}, nodes[0].chain_monitor.clone(), nodes[0].node.clone(),
+                       nodes[0].rapid_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(),
+                       Some(nodes[0].scorer.clone()), move |dur: Duration| {
+                               let mut exit_receiver = exit_receiver.clone();
+                               Box::pin(async move {
+                                       tokio::select! {
+                                               _ = tokio::time::sleep(dur) => false,
+                                               _ = exit_receiver.changed() => true,
+                                       }
+                               })
+                       }, false,
+               );
+
+               let t1 = tokio::spawn(bp_future);
+               let t2 = tokio::spawn(async move {
+                       do_test_not_pruning_network_graph_until_graph_sync_completion!(nodes, {
+                               let mut i = 0;
+                               loop {
+                                       tokio::time::sleep(Duration::from_secs(super::FIRST_NETWORK_PRUNE_TIMER)).await;
+                                       if let Ok(()) = receiver.try_recv() { break Ok::<(), ()>(()); }
+                                       assert!(i < 5);
+                                       i += 1;
+                               }
+                       }, tokio::time::sleep(Duration::from_millis(1)).await);
+                       exit_sender.send(()).unwrap();
+               });
+               let (r1, r2) = tokio::join!(t1, t2);
+               r1.unwrap().unwrap();
+               r2.unwrap()
+       }
+
+       macro_rules! do_test_payment_path_scoring {
+               ($nodes: expr, $receive: expr) => {
+                       // Ensure that we update the scorer when relevant events are processed. In this case, we ensure
+                       // that we update the scorer upon a payment path succeeding (note that the channel must be
+                       // public or else we won't score it).
+                       // A background event handler for FundingGenerationReady events must be hooked up to a
+                       // running background processor.
+                       let scored_scid = 4242;
+                       let secp_ctx = Secp256k1::new();
+                       let node_1_privkey = SecretKey::from_slice(&[42; 32]).unwrap();
+                       let node_1_id = PublicKey::from_secret_key(&secp_ctx, &node_1_privkey);
+
+                       let path = Path { hops: vec![RouteHop {
+                               pubkey: node_1_id,
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: scored_scid,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 0,
+                               cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA as u32,
+                       }], blinded_tail: None };
+
+                       $nodes[0].scorer.lock().unwrap().expect(TestResult::PaymentFailure { path: path.clone(), short_channel_id: scored_scid });
+                       $nodes[0].node.push_pending_event(Event::PaymentPathFailed {
+                               payment_id: None,
+                               payment_hash: PaymentHash([42; 32]),
+                               payment_failed_permanently: false,
+                               failure: PathFailure::OnPath { network_update: None },
+                               path: path.clone(),
+                               short_channel_id: Some(scored_scid),
+                       });
+                       let event = $receive.expect("PaymentPathFailed not handled within deadline");
+                       match event {
+                               Event::PaymentPathFailed { .. } => {},
+                               _ => panic!("Unexpected event"),
                        }
-               }
 
-               let initialization_input = vec![
-                       76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
-                       79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
-                       0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
-                       187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
-                       157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
-                       88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
-                       204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
-                       181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
-                       110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
-                       76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
-                       226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 2, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
-                       0, 0, 0, 1, 0, 0, 0, 0, 58, 85, 116, 216, 255, 8, 153, 192, 0, 2, 27, 0, 0, 25, 0, 0,
-                       0, 1, 0, 0, 0, 125, 255, 2, 68, 226, 0, 6, 11, 0, 1, 5, 0, 0, 0, 0, 29, 129, 25, 192,
-               ];
-               nodes[0].rapid_gossip_sync.update_network_graph_no_std(&initialization_input[..], Some(1642291930)).unwrap();
-
-               // this should have added two channels
-               assert_eq!(network_graph.read_only().channels().len(), 3);
-
-               receiver
-                       .recv_timeout(Duration::from_secs(super::FIRST_NETWORK_PRUNE_TIMER * 5))
-                       .expect("Network graph not pruned within deadline");
+                       // Ensure we'll score payments that were explicitly failed back by the destination as
+                       // ProbeSuccess.
+                       $nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeSuccess { path: path.clone() });
+                       $nodes[0].node.push_pending_event(Event::PaymentPathFailed {
+                               payment_id: None,
+                               payment_hash: PaymentHash([42; 32]),
+                               payment_failed_permanently: true,
+                               failure: PathFailure::OnPath { network_update: None },
+                               path: path.clone(),
+                               short_channel_id: None,
+                       });
+                       let event = $receive.expect("PaymentPathFailed not handled within deadline");
+                       match event {
+                               Event::PaymentPathFailed { .. } => {},
+                               _ => panic!("Unexpected event"),
+                       }
 
-               background_processor.stop().unwrap();
+                       $nodes[0].scorer.lock().unwrap().expect(TestResult::PaymentSuccess { path: path.clone() });
+                       $nodes[0].node.push_pending_event(Event::PaymentPathSuccessful {
+                               payment_id: PaymentId([42; 32]),
+                               payment_hash: None,
+                               path: path.clone(),
+                       });
+                       let event = $receive.expect("PaymentPathSuccessful not handled within deadline");
+                       match event {
+                               Event::PaymentPathSuccessful { .. } => {},
+                               _ => panic!("Unexpected event"),
+                       }
+
+                       $nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeSuccess { path: path.clone() });
+                       $nodes[0].node.push_pending_event(Event::ProbeSuccessful {
+                               payment_id: PaymentId([42; 32]),
+                               payment_hash: PaymentHash([42; 32]),
+                               path: path.clone(),
+                       });
+                       let event = $receive.expect("ProbeSuccessful not handled within deadline");
+                       match event {
+                               Event::ProbeSuccessful  { .. } => {},
+                               _ => panic!("Unexpected event"),
+                       }
 
-               // all channels should now be pruned
-               assert_eq!(network_graph.read_only().channels().len(), 0);
+                       $nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeFailure { path: path.clone() });
+                       $nodes[0].node.push_pending_event(Event::ProbeFailed {
+                               payment_id: PaymentId([42; 32]),
+                               payment_hash: PaymentHash([42; 32]),
+                               path,
+                               short_channel_id: Some(scored_scid),
+                       });
+                       let event = $receive.expect("ProbeFailure not handled within deadline");
+                       match event {
+                               Event::ProbeFailed { .. } => {},
+                               _ => panic!("Unexpected event"),
+                       }
+               }
        }
 
        #[test]
        fn test_payment_path_scoring() {
-               // Ensure that we update the scorer when relevant events are processed. In this case, we ensure
-               // that we update the scorer upon a payment path succeeding (note that the channel must be
-               // public or else we won't score it).
-               // Set up a background event handler for FundingGenerationReady events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
                let event_handler = move |event: Event| match event {
                        Event::PaymentPathFailed { .. } => sender.send(event).unwrap(),
@@ -1388,104 +1710,62 @@ mod tests {
                        _ => panic!("Unexpected event: {:?}", event),
                };
 
-               let nodes = create_nodes(1, "test_payment_path_scoring".to_string());
+               let (_, nodes) = create_nodes(1, "test_payment_path_scoring");
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
-               let scored_scid = 4242;
-               let secp_ctx = Secp256k1::new();
-               let node_1_privkey = SecretKey::from_slice(&[42; 32]).unwrap();
-               let node_1_id = PublicKey::from_secret_key(&secp_ctx, &node_1_privkey);
-
-               let path = vec![RouteHop {
-                       pubkey: node_1_id,
-                       node_features: NodeFeatures::empty(),
-                       short_channel_id: scored_scid,
-                       channel_features: ChannelFeatures::empty(),
-                       fee_msat: 0,
-                       cltv_expiry_delta: MIN_CLTV_EXPIRY_DELTA as u32,
-               }];
-
-               nodes[0].scorer.lock().unwrap().expect(TestResult::PaymentFailure { path: path.clone(), short_channel_id: scored_scid });
-               nodes[0].node.push_pending_event(Event::PaymentPathFailed {
-                       payment_id: None,
-                       payment_hash: PaymentHash([42; 32]),
-                       payment_failed_permanently: false,
-                       failure: PathFailure::OnPath { network_update: None },
-                       path: path.clone(),
-                       short_channel_id: Some(scored_scid),
-               });
-               let event = receiver
-                       .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("PaymentPathFailed not handled within deadline");
-               match event {
-                       Event::PaymentPathFailed { .. } => {},
-                       _ => panic!("Unexpected event"),
-               }
+               do_test_payment_path_scoring!(nodes, receiver.recv_timeout(Duration::from_secs(EVENT_DEADLINE)));
 
-               // Ensure we'll score payments that were explicitly failed back by the destination as
-               // ProbeSuccess.
-               nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeSuccess { path: path.clone() });
-               nodes[0].node.push_pending_event(Event::PaymentPathFailed {
-                       payment_id: None,
-                       payment_hash: PaymentHash([42; 32]),
-                       payment_failed_permanently: true,
-                       failure: PathFailure::OnPath { network_update: None },
-                       path: path.clone(),
-                       short_channel_id: None,
-               });
-               let event = receiver
-                       .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("PaymentPathFailed not handled within deadline");
-               match event {
-                       Event::PaymentPathFailed { .. } => {},
-                       _ => panic!("Unexpected event"),
+               if !std::thread::panicking() {
+                       bg_processor.stop().unwrap();
                }
+       }
 
-               nodes[0].scorer.lock().unwrap().expect(TestResult::PaymentSuccess { path: path.clone() });
-               nodes[0].node.push_pending_event(Event::PaymentPathSuccessful {
-                       payment_id: PaymentId([42; 32]),
-                       payment_hash: None,
-                       path: path.clone(),
-               });
-               let event = receiver
-                       .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("PaymentPathSuccessful not handled within deadline");
-               match event {
-                       Event::PaymentPathSuccessful { .. } => {},
-                       _ => panic!("Unexpected event"),
-               }
+       #[tokio::test]
+       #[cfg(feature = "futures")]
+       async fn test_payment_path_scoring_async() {
+               let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
+               let event_handler = move |event: Event| {
+                       let sender_ref = sender.clone();
+                       async move {
+                               match event {
+                                       Event::PaymentPathFailed { .. } => { sender_ref.send(event).await.unwrap() },
+                                       Event::PaymentPathSuccessful { .. } => { sender_ref.send(event).await.unwrap() },
+                                       Event::ProbeSuccessful { .. } => { sender_ref.send(event).await.unwrap() },
+                                       Event::ProbeFailed { .. } => { sender_ref.send(event).await.unwrap() },
+                                       _ => panic!("Unexpected event: {:?}", event),
+                               }
+                       }
+               };
 
-               nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeSuccess { path: path.clone() });
-               nodes[0].node.push_pending_event(Event::ProbeSuccessful {
-                       payment_id: PaymentId([42; 32]),
-                       payment_hash: PaymentHash([42; 32]),
-                       path: path.clone(),
-               });
-               let event = receiver
-                       .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("ProbeSuccessful not handled within deadline");
-               match event {
-                       Event::ProbeSuccessful  { .. } => {},
-                       _ => panic!("Unexpected event"),
-               }
+               let (_, nodes) = create_nodes(1, "test_payment_path_scoring_async");
+               let data_dir = nodes[0].persister.get_data_dir();
+               let persister = Arc::new(Persister::new(data_dir));
 
-               nodes[0].scorer.lock().unwrap().expect(TestResult::ProbeFailure { path: path.clone() });
-               nodes[0].node.push_pending_event(Event::ProbeFailed {
-                       payment_id: PaymentId([42; 32]),
-                       payment_hash: PaymentHash([42; 32]),
-                       path,
-                       short_channel_id: Some(scored_scid),
+               let (exit_sender, exit_receiver) = tokio::sync::watch::channel(());
+
+               let bp_future = super::process_events_async(
+                       persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(),
+                       nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(),
+                       Some(nodes[0].scorer.clone()), move |dur: Duration| {
+                               let mut exit_receiver = exit_receiver.clone();
+                               Box::pin(async move {
+                                       tokio::select! {
+                                               _ = tokio::time::sleep(dur) => false,
+                                               _ = exit_receiver.changed() => true,
+                                       }
+                               })
+                       }, false,
+               );
+               let t1 = tokio::spawn(bp_future);
+               let t2 = tokio::spawn(async move {
+                       do_test_payment_path_scoring!(nodes, receiver.recv().await);
+                       exit_sender.send(()).unwrap();
                });
-               let event = receiver
-                       .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("ProbeFailure not handled within deadline");
-               match event {
-                       Event::ProbeFailed { .. } => {},
-                       _ => panic!("Unexpected event"),
-               }
 
-               assert!(bg_processor.stop().is_ok());
+               let (r1, r2) = tokio::join!(t1, t2);
+               r1.unwrap().unwrap();
+               r2.unwrap()
        }
 }
index 59f8c2356057c32cd7ec01bf251569e230b82d72..a19c3ff8a9dfc41640d325763c0056b6740fc046 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-block-sync"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Jeffrey Czyz", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -19,11 +19,11 @@ rpc-client = [ "serde_json", "chunked_transfer" ]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.114", path = "../lightning" }
+lightning = { version = "0.0.115", path = "../lightning" }
 tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true }
 serde_json = { version = "1.0", optional = true }
 chunked_transfer = { version = "1.4", optional = true }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }
 tokio = { version = "1.14", features = [ "macros", "rt" ] }
index 509b6024d02c9086f3b9383a0500c050bb84d5a9..68aa2a1cb259393545c7637d4a8f08ef4bcce6a9 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-custom-message"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Jeffrey Czyz"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -15,4 +15,4 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.114", path = "../lightning" }
+lightning = { version = "0.0.115", path = "../lightning" }
index a3fb66a8f91425fc8459a38144023cfb8e9fe84e..5179fdc148a5158e9e9e91d64371a09a243ada11 100644 (file)
@@ -1,7 +1,7 @@
 [package]
 name = "lightning-invoice"
 description = "Data structures to parse and serialize BOLT11 lightning invoices"
-version = "0.22.0"
+version = "0.23.0"
 authors = ["Sebastian Geisler <sgeisler@wh2.tu-dresden.de>"]
 documentation = "https://docs.rs/lightning-invoice/"
 license = "MIT OR Apache-2.0"
@@ -21,14 +21,15 @@ std = ["bitcoin_hashes/std", "num-traits/std", "lightning/std", "bech32/std"]
 
 [dependencies]
 bech32 = { version = "0.9.0", default-features = false }
-lightning = { version = "0.0.114", path = "../lightning", default-features = false }
+lightning = { version = "0.0.115", path = "../lightning", default-features = false }
 secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"] }
 num-traits = { version = "0.2.8", default-features = false }
 bitcoin_hashes = { version = "0.11", default-features = false }
 hashbrown = { version = "0.8", optional = true }
 serde = { version = "1.0.118", optional = true }
+bitcoin = { version = "0.29.0", default-features = false }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", default-features = false, features = ["_test_utils"] }
+lightning = { version = "0.0.115", path = "../lightning", default-features = false, features = ["_test_utils"] }
 hex = "0.4"
 serde_json = { version = "1"}
index ab9fd9cb2bc21446920360759feefec87291f774..01adf67d1af0816bcc78ddeeacd6473f680b71bf 100644 (file)
@@ -1,5 +1,6 @@
 #[cfg(feature = "std")]
 use std::error;
+use core::convert::TryFrom;
 use core::fmt;
 use core::fmt::{Display, Formatter};
 use core::num::ParseIntError;
@@ -8,6 +9,8 @@ use core::str::FromStr;
 
 use bech32::{u5, FromBase32};
 
+use bitcoin::{PubkeyHash, ScriptHash};
+use bitcoin::util::address::WitnessVersion;
 use bitcoin_hashes::Hash;
 use bitcoin_hashes::sha256;
 use crate::prelude::*;
@@ -457,6 +460,8 @@ impl FromBase32 for TaggedField {
                                Ok(TaggedField::PrivateRoute(PrivateRoute::from_base32(field_data)?)),
                        constants::TAG_PAYMENT_SECRET =>
                                Ok(TaggedField::PaymentSecret(PaymentSecret::from_base32(field_data)?)),
+                       constants::TAG_PAYMENT_METADATA =>
+                               Ok(TaggedField::PaymentMetadata(Vec::<u8>::from_base32(field_data)?)),
                        constants::TAG_FEATURES =>
                                Ok(TaggedField::Features(InvoiceFeatures::from_base32(field_data)?)),
                        _ => {
@@ -550,27 +555,24 @@ impl FromBase32 for Fallback {
                                if bytes.len() < 2 || bytes.len() > 40 {
                                        return Err(ParseError::InvalidSegWitProgramLength);
                                }
-
+                               let version = WitnessVersion::try_from(version).expect("0 through 16 are valid SegWit versions");
                                Ok(Fallback::SegWitProgram {
                                        version,
                                        program: bytes
                                })
                        },
                        17 => {
-                               if bytes.len() != 20 {
-                                       return Err(ParseError::InvalidPubKeyHashLength);
-                               }
-                               //TODO: refactor once const generics are available
-                               let mut pkh = [0u8; 20];
-                               pkh.copy_from_slice(&bytes);
+                               let pkh = match PubkeyHash::from_slice(&bytes) {
+                                       Ok(pkh) => pkh,
+                                       Err(bitcoin_hashes::Error::InvalidLength(_, _)) => return Err(ParseError::InvalidPubKeyHashLength),
+                               };
                                Ok(Fallback::PubKeyHash(pkh))
                        }
                        18 => {
-                               if bytes.len() != 20 {
-                                       return Err(ParseError::InvalidScriptHashLength);
-                               }
-                               let mut sh = [0u8; 20];
-                               sh.copy_from_slice(&bytes);
+                               let sh = match ScriptHash::from_slice(&bytes) {
+                                       Ok(sh) => sh,
+                                       Err(bitcoin_hashes::Error::InvalidLength(_, _)) => return Err(ParseError::InvalidScriptHashLength),
+                               };
                                Ok(Fallback::ScriptHash(sh))
                        }
                        _ => Err(ParseError::Skip)
@@ -852,26 +854,29 @@ mod test {
        fn test_parse_fallback() {
                use crate::Fallback;
                use bech32::FromBase32;
+               use bitcoin::{PubkeyHash, ScriptHash};
+               use bitcoin::util::address::WitnessVersion;
+               use bitcoin_hashes::Hash;
 
                let cases = vec![
                        (
                                from_bech32("3x9et2e20v6pu37c5d9vax37wxq72un98".as_bytes()),
-                               Ok(Fallback::PubKeyHash([
+                               Ok(Fallback::PubKeyHash(PubkeyHash::from_slice(&[
                                        0x31, 0x72, 0xb5, 0x65, 0x4f, 0x66, 0x83, 0xc8, 0xfb, 0x14, 0x69, 0x59, 0xd3,
                                        0x47, 0xce, 0x30, 0x3c, 0xae, 0x4c, 0xa7
-                               ]))
+                               ]).unwrap()))
                        ),
                        (
                                from_bech32("j3a24vwu6r8ejrss3axul8rxldph2q7z9".as_bytes()),
-                               Ok(Fallback::ScriptHash([
+                               Ok(Fallback::ScriptHash(ScriptHash::from_slice(&[
                                        0x8f, 0x55, 0x56, 0x3b, 0x9a, 0x19, 0xf3, 0x21, 0xc2, 0x11, 0xe9, 0xb9, 0xf3,
                                        0x8c, 0xdf, 0x68, 0x6e, 0xa0, 0x78, 0x45
-                               ]))
+                               ]).unwrap()))
                        ),
                        (
                                from_bech32("qw508d6qejxtdg4y5r3zarvary0c5xw7k".as_bytes()),
                                Ok(Fallback::SegWitProgram {
-                                       version: u5::try_from_u8(0).unwrap(),
+                                       version: WitnessVersion::V0,
                                        program: Vec::from(&[
                                                0x75u8, 0x1e, 0x76, 0xe8, 0x19, 0x91, 0x96, 0xd4, 0x54, 0x94, 0x1c, 0x45,
                                                0xd1, 0xb3, 0xa3, 0x23, 0xf1, 0x43, 0x3b, 0xd6
index 593701cab458c85079dff3e135dcebbf63845277..101051b5e7f23509325d42301a424d3045fc7589 100644 (file)
 //! invoices and functions to create, encode and decode these. If you just want to use the standard
 //! en-/decoding functionality this should get you started:
 //!
-//!   * For parsing use `str::parse::<Invoice>(&self)` (see the docs of `impl FromStr for Invoice`)
-//!   * For constructing invoices use the `InvoiceBuilder`
-//!   * For serializing invoices use the `Display`/`ToString` traits
+//!   * For parsing use `str::parse::<Invoice>(&self)` (see [`Invoice::from_str`])
+//!   * For constructing invoices use the [`InvoiceBuilder`]
+//!   * For serializing invoices use the [`Display`]/[`ToString`] traits
+//!
+//! [`Invoice::from_str`]: crate::Invoice#impl-FromStr
 
 #[cfg(not(any(feature = "std", feature = "no-std")))]
 compile_error!("at least one of the `std` or `no-std` features must be enabled");
@@ -45,8 +47,9 @@ extern crate serde;
 use std::time::SystemTime;
 
 use bech32::u5;
-use bitcoin_hashes::Hash;
-use bitcoin_hashes::sha256;
+use bitcoin::{Address, Network, PubkeyHash, ScriptHash};
+use bitcoin::util::address::{Payload, WitnessVersion};
+use bitcoin_hashes::{Hash, sha256};
 use lightning::ln::PaymentSecret;
 use lightning::ln::features::InvoiceFeatures;
 #[cfg(any(doc, test))]
@@ -160,7 +163,7 @@ pub const DEFAULT_EXPIRY_TIME: u64 = 3600;
 /// [`MIN_FINAL_CLTV_EXPIRY_DELTA`]: lightning::ln::channelmanager::MIN_FINAL_CLTV_EXPIRY_DELTA
 pub const DEFAULT_MIN_FINAL_CLTV_EXPIRY_DELTA: u64 = 18;
 
-/// Builder for `Invoice`s. It's the most convenient and advised way to use this library. It ensures
+/// Builder for [`Invoice`]s. It's the most convenient and advised way to use this library. It ensures
 /// that only a semantically and syntactically correct Invoice can be built using it.
 ///
 /// ```
@@ -212,13 +215,16 @@ pub const DEFAULT_MIN_FINAL_CLTV_EXPIRY_DELTA: u64 = 18;
 /// # Type parameters
 /// The two parameters `D` and `H` signal if the builder already contains the correct amount of the
 /// given field:
-///  * `D`: exactly one `Description` or `DescriptionHash`
-///  * `H`: exactly one `PaymentHash`
+///  * `D`: exactly one [`TaggedField::Description`] or [`TaggedField::DescriptionHash`]
+///  * `H`: exactly one [`TaggedField::PaymentHash`]
 ///  * `T`: the timestamp is set
+///  * `C`: the CLTV expiry is set
+///  * `S`: the payment secret is set
+///  * `M`: payment metadata is set
 ///
 /// This is not exported to bindings users as we likely need to manually select one set of boolean type parameters.
 #[derive(Eq, PartialEq, Debug, Clone)]
-pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> {
+pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> {
        currency: Currency,
        amount: Option<u64>,
        si_prefix: Option<SiPrefix>,
@@ -231,14 +237,17 @@ pub struct InvoiceBuilder<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S:
        phantom_t: core::marker::PhantomData<T>,
        phantom_c: core::marker::PhantomData<C>,
        phantom_s: core::marker::PhantomData<S>,
+       phantom_m: core::marker::PhantomData<M>,
 }
 
 /// Represents a syntactically and semantically correct lightning BOLT11 invoice.
 ///
 /// There are three ways to construct an `Invoice`:
-///  1. using `InvoiceBuilder`
-///  2. using `Invoice::from_signed(SignedRawInvoice)`
-///  3. using `str::parse::<Invoice>(&str)`
+///  1. using [`InvoiceBuilder`]
+///  2. using [`Invoice::from_signed`]
+///  3. using `str::parse::<Invoice>(&str)` (see [`Invoice::from_str`])
+///
+/// [`Invoice::from_str`]: crate::Invoice#impl-FromStr
 #[derive(Eq, PartialEq, Debug, Clone, Hash)]
 pub struct Invoice {
        signed_invoice: SignedRawInvoice,
@@ -258,34 +267,34 @@ pub enum InvoiceDescription<'f> {
        Hash(&'f Sha256),
 }
 
-/// Represents a signed `RawInvoice` with cached hash. The signature is not checked and may be
+/// Represents a signed [`RawInvoice`] with cached hash. The signature is not checked and may be
 /// invalid.
 ///
 /// # Invariants
-/// The hash has to be either from the deserialized invoice or from the serialized `raw_invoice`.
+/// The hash has to be either from the deserialized invoice or from the serialized [`RawInvoice`].
 #[derive(Eq, PartialEq, Debug, Clone, Hash)]
 pub struct SignedRawInvoice {
        /// The rawInvoice that the signature belongs to
        raw_invoice: RawInvoice,
 
-       /// Hash of the `RawInvoice` that will be used to check the signature.
+       /// Hash of the [`RawInvoice`] that will be used to check the signature.
        ///
        /// * if the `SignedRawInvoice` was deserialized the hash is of from the original encoded form,
        /// since it's not guaranteed that encoding it again will lead to the same result since integers
        /// could have been encoded with leading zeroes etc.
        /// * if the `SignedRawInvoice` was constructed manually the hash will be the calculated hash
-       /// from the `RawInvoice`
+       /// from the [`RawInvoice`]
        hash: [u8; 32],
 
        /// signature of the payment request
        signature: InvoiceSignature,
 }
 
-/// Represents an syntactically correct Invoice for a payment on the lightning network,
+/// Represents an syntactically correct [`Invoice`] for a payment on the lightning network,
 /// but without the signature information.
-/// De- and encoding should not lead to information loss but may lead to different hashes.
+/// Decoding and encoding should not lead to information loss but may lead to different hashes.
 ///
-/// For methods without docs see the corresponding methods in `Invoice`.
+/// For methods without docs see the corresponding methods in [`Invoice`].
 #[derive(Eq, PartialEq, Debug, Clone, Hash)]
 pub struct RawInvoice {
        /// human readable part
@@ -295,7 +304,7 @@ pub struct RawInvoice {
        pub data: RawDataPart,
 }
 
-/// Data of the `RawInvoice` that is encoded in the human readable part
+/// Data of the [`RawInvoice`] that is encoded in the human readable part.
 ///
 /// This is not exported to bindings users as we don't yet support `Option<Enum>`
 #[derive(Eq, PartialEq, Debug, Clone, Hash)]
@@ -310,7 +319,7 @@ pub struct RawHrp {
        pub si_prefix: Option<SiPrefix>,
 }
 
-/// Data of the `RawInvoice` that is encoded in the data part
+/// Data of the [`RawInvoice`] that is encoded in the data part
 #[derive(Eq, PartialEq, Debug, Clone, Hash)]
 pub struct RawDataPart {
        /// generation time of the invoice
@@ -385,6 +394,29 @@ pub enum Currency {
        Signet,
 }
 
+impl From<Network> for Currency {
+       fn from(network: Network) -> Self {
+               match network {
+                       Network::Bitcoin => Currency::Bitcoin,
+                       Network::Testnet => Currency::BitcoinTestnet,
+                       Network::Regtest => Currency::Regtest,
+                       Network::Signet => Currency::Signet,
+               }
+       }
+}
+
+impl From<Currency> for Network {
+       fn from(currency: Currency) -> Self {
+               match currency {
+                       Currency::Bitcoin => Network::Bitcoin,
+                       Currency::BitcoinTestnet => Network::Testnet,
+                       Currency::Regtest => Network::Regtest,
+                       Currency::Simnet => Network::Regtest,
+                       Currency::Signet => Network::Signet,
+               }
+       }
+}
+
 /// Tagged field which may have an unknown tag
 ///
 /// This is not exported to bindings users as we don't currently support TaggedField
@@ -414,6 +446,7 @@ pub enum TaggedField {
        Fallback(Fallback),
        PrivateRoute(PrivateRoute),
        PaymentSecret(PaymentSecret),
+       PaymentMetadata(Vec<u8>),
        Features(InvoiceFeatures),
 }
 
@@ -422,6 +455,15 @@ pub enum TaggedField {
 pub struct Sha256(/// This is not exported to bindings users as the native hash types are not currently mapped
        pub sha256::Hash);
 
+impl Sha256 {
+       /// Constructs a new [`Sha256`] from the given bytes, which are assumed to be the output of a
+       /// single sha256 hash.
+       #[cfg(c_bindings)]
+       pub fn from_bytes(bytes: &[u8; 32]) -> Self {
+               Self(sha256::Hash::from_slice(bytes).expect("from_slice only fails if len is not 32"))
+       }
+}
+
 /// Description string
 ///
 /// # Invariants
@@ -442,17 +484,16 @@ pub struct ExpiryTime(Duration);
 #[derive(Clone, Debug, Hash, Eq, PartialEq)]
 pub struct MinFinalCltvExpiryDelta(pub u64);
 
-// TODO: better types instead onf byte arrays
 /// Fallback address in case no LN payment is possible
 #[allow(missing_docs)]
 #[derive(Clone, Debug, Hash, Eq, PartialEq)]
 pub enum Fallback {
        SegWitProgram {
-               version: u5,
+               version: WitnessVersion,
                program: Vec<u8>,
        },
-       PubKeyHash([u8; 20]),
-       ScriptHash([u8; 20]),
+       PubKeyHash(PubkeyHash),
+       ScriptHash(ScriptHash),
 }
 
 /// Recoverable signature
@@ -479,15 +520,16 @@ pub mod constants {
        pub const TAG_FALLBACK: u8 = 9;
        pub const TAG_PRIVATE_ROUTE: u8 = 3;
        pub const TAG_PAYMENT_SECRET: u8 = 16;
+       pub const TAG_PAYMENT_METADATA: u8 = 27;
        pub const TAG_FEATURES: u8 = 5;
 }
 
-impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False> {
+impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False, tb::False> {
        /// Construct new, empty `InvoiceBuilder`. All necessary fields have to be filled first before
        /// `InvoiceBuilder::build(self)` becomes available.
-       pub fn new(currrency: Currency) -> Self {
+       pub fn new(currency: Currency) -> Self {
                InvoiceBuilder {
-                       currency: currrency,
+                       currency,
                        amount: None,
                        si_prefix: None,
                        timestamp: None,
@@ -499,14 +541,15 @@ impl InvoiceBuilder<tb::False, tb::False, tb::False, tb::False, tb::False> {
                        phantom_t: core::marker::PhantomData,
                        phantom_c: core::marker::PhantomData,
                        phantom_s: core::marker::PhantomData,
+                       phantom_m: core::marker::PhantomData,
                }
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, S, M> {
        /// Helper function to set the completeness flags.
-       fn set_flags<DN: tb::Bool, HN: tb::Bool, TN: tb::Bool, CN: tb::Bool, SN: tb::Bool>(self) -> InvoiceBuilder<DN, HN, TN, CN, SN> {
-               InvoiceBuilder::<DN, HN, TN, CN, SN> {
+       fn set_flags<DN: tb::Bool, HN: tb::Bool, TN: tb::Bool, CN: tb::Bool, SN: tb::Bool, MN: tb::Bool>(self) -> InvoiceBuilder<DN, HN, TN, CN, SN, MN> {
+               InvoiceBuilder::<DN, HN, TN, CN, SN, MN> {
                        currency: self.currency,
                        amount: self.amount,
                        si_prefix: self.si_prefix,
@@ -519,6 +562,7 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBui
                        phantom_t: core::marker::PhantomData,
                        phantom_c: core::marker::PhantomData,
                        phantom_s: core::marker::PhantomData,
+                       phantom_m: core::marker::PhantomData,
                }
        }
 
@@ -563,8 +607,9 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBui
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb::True, C, S> {
-       /// Builds a `RawInvoice` if no `CreationError` occurred while construction any of the fields.
+impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, tb::True, C, S, M> {
+       /// Builds a [`RawInvoice`] if no [`CreationError`] occurred while construction any of the
+       /// fields.
        pub fn build_raw(self) -> Result<RawInvoice, CreationError> {
 
                // If an error occurred at any time before, return it now
@@ -596,9 +641,9 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
        }
 }
 
-impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::False, H, T, C, S> {
+impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<tb::False, H, T, C, S, M> {
        /// Set the description. This function is only available if no description (hash) was set.
-       pub fn description(mut self, description: String) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn description(mut self, description: String) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                match Description::new(description) {
                        Ok(d) => self.tagged_fields.push(TaggedField::Description(d)),
                        Err(e) => self.error = Some(e),
@@ -607,13 +652,13 @@ impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::Fals
        }
 
        /// Set the description hash. This function is only available if no description (hash) was set.
-       pub fn description_hash(mut self, description_hash: sha256::Hash) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn description_hash(mut self, description_hash: sha256::Hash) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                self.tagged_fields.push(TaggedField::DescriptionHash(Sha256(description_hash)));
                self.set_flags()
        }
 
        /// Set the description or description hash. This function is only available if no description (hash) was set.
-       pub fn invoice_description(self, description: InvoiceDescription) -> InvoiceBuilder<tb::True, H, T, C, S> {
+       pub fn invoice_description(self, description: InvoiceDescription) -> InvoiceBuilder<tb::True, H, T, C, S, M> {
                match description {
                        InvoiceDescription::Direct(desc) => {
                                self.description(desc.clone().into_inner())
@@ -625,18 +670,18 @@ impl<H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<tb::Fals
        }
 }
 
-impl<D: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, tb::False, T, C, S> {
+impl<D: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, tb::False, T, C, S, M> {
        /// Set the payment hash. This function is only available if no payment hash was set.
-       pub fn payment_hash(mut self, hash: sha256::Hash) -> InvoiceBuilder<D, tb::True, T, C, S> {
+       pub fn payment_hash(mut self, hash: sha256::Hash) -> InvoiceBuilder<D, tb::True, T, C, S, M> {
                self.tagged_fields.push(TaggedField::PaymentHash(Sha256(hash)));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb::False, C, S> {
+impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, tb::False, C, S, M> {
        /// Sets the timestamp to a specific [`SystemTime`].
        #[cfg(feature = "std")]
-       pub fn timestamp(mut self, time: SystemTime) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn timestamp(mut self, time: SystemTime) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                match PositiveTimestamp::from_system_time(time) {
                        Ok(t) => self.timestamp = Some(t),
                        Err(e) => self.error = Some(e),
@@ -647,7 +692,7 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
 
        /// Sets the timestamp to a duration since the Unix epoch, dropping the subsecond part (which
        /// is not representable in BOLT 11 invoices).
-       pub fn duration_since_epoch(mut self, time: Duration) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn duration_since_epoch(mut self, time: Duration) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                match PositiveTimestamp::from_duration_since_epoch(time) {
                        Ok(t) => self.timestamp = Some(t),
                        Err(e) => self.error = Some(e),
@@ -658,34 +703,82 @@ impl<D: tb::Bool, H: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, tb
 
        /// Sets the timestamp to the current system time.
        #[cfg(feature = "std")]
-       pub fn current_timestamp(mut self) -> InvoiceBuilder<D, H, tb::True, C, S> {
+       pub fn current_timestamp(mut self) -> InvoiceBuilder<D, H, tb::True, C, S, M> {
                let now = PositiveTimestamp::from_system_time(SystemTime::now());
                self.timestamp = Some(now.expect("for the foreseeable future this shouldn't happen"));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, tb::False, S> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, S: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, tb::False, S, M> {
        /// Sets `min_final_cltv_expiry_delta`.
-       pub fn min_final_cltv_expiry_delta(mut self, min_final_cltv_expiry_delta: u64) -> InvoiceBuilder<D, H, T, tb::True, S> {
+       pub fn min_final_cltv_expiry_delta(mut self, min_final_cltv_expiry_delta: u64) -> InvoiceBuilder<D, H, T, tb::True, S, M> {
                self.tagged_fields.push(TaggedField::MinFinalCltvExpiryDelta(MinFinalCltvExpiryDelta(min_final_cltv_expiry_delta)));
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T, C, tb::False> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, tb::False, M> {
        /// Sets the payment secret and relevant features.
-       pub fn payment_secret(mut self, payment_secret: PaymentSecret) -> InvoiceBuilder<D, H, T, C, tb::True> {
-               let mut features = InvoiceFeatures::empty();
-               features.set_variable_length_onion_required();
-               features.set_payment_secret_required();
+       pub fn payment_secret(mut self, payment_secret: PaymentSecret) -> InvoiceBuilder<D, H, T, C, tb::True, M> {
+               let mut found_features = false;
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               found_features = true;
+                               f.set_variable_length_onion_required();
+                               f.set_payment_secret_required();
+                       }
+               }
                self.tagged_fields.push(TaggedField::PaymentSecret(payment_secret));
-               self.tagged_fields.push(TaggedField::Features(features));
+               if !found_features {
+                       let mut features = InvoiceFeatures::empty();
+                       features.set_variable_length_onion_required();
+                       features.set_payment_secret_required();
+                       self.tagged_fields.push(TaggedField::Features(features));
+               }
                self.set_flags()
        }
 }
 
-impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T, C, tb::True> {
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S, tb::False> {
+       /// Sets the payment metadata.
+       ///
+       /// By default features are set to *optionally* allow the sender to include the payment metadata.
+       /// If you wish to require that the sender include the metadata (and fail to parse the invoice if
+       /// they don't support payment metadata fields), you need to call
+       /// [`InvoiceBuilder::require_payment_metadata`] after this.
+       pub fn payment_metadata(mut self, payment_metadata: Vec<u8>) -> InvoiceBuilder<D, H, T, C, S, tb::True> {
+               self.tagged_fields.push(TaggedField::PaymentMetadata(payment_metadata));
+               let mut found_features = false;
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               found_features = true;
+                               f.set_payment_metadata_optional();
+                       }
+               }
+               if !found_features {
+                       let mut features = InvoiceFeatures::empty();
+                       features.set_payment_metadata_optional();
+                       self.tagged_fields.push(TaggedField::Features(features));
+               }
+               self.set_flags()
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, S: tb::Bool> InvoiceBuilder<D, H, T, C, S, tb::True> {
+       /// Sets forwarding of payment metadata as required. A reader of the invoice which does not
+       /// support sending payment metadata will fail to read the invoice.
+       pub fn require_payment_metadata(mut self) -> InvoiceBuilder<D, H, T, C, S, tb::True> {
+               for field in self.tagged_fields.iter_mut() {
+                       if let TaggedField::Features(f) = field {
+                               f.set_payment_metadata_required();
+                       }
+               }
+               self
+       }
+}
+
+impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool, M: tb::Bool> InvoiceBuilder<D, H, T, C, tb::True, M> {
        /// Sets the `basic_mpp` feature as optional.
        pub fn basic_mpp(mut self) -> Self {
                for field in self.tagged_fields.iter_mut() {
@@ -697,7 +790,7 @@ impl<D: tb::Bool, H: tb::Bool, T: tb::Bool, C: tb::Bool> InvoiceBuilder<D, H, T,
        }
 }
 
-impl InvoiceBuilder<tb::True, tb::True, tb::True, tb::True, tb::True> {
+impl<M: tb::Bool> InvoiceBuilder<tb::True, tb::True, tb::True, tb::True, tb::True, M> {
        /// Builds and signs an invoice using the supplied `sign_function`. This function MAY NOT fail
        /// and MUST produce a recoverable signature valid for the given hash and if applicable also for
        /// the included payee public key.
@@ -753,17 +846,17 @@ impl SignedRawInvoice {
                (self.raw_invoice, self.hash, self.signature)
        }
 
-       /// The `RawInvoice` which was signed.
+       /// The [`RawInvoice`] which was signed.
        pub fn raw_invoice(&self) -> &RawInvoice {
                &self.raw_invoice
        }
 
-       /// The hash of the `RawInvoice` that was signed.
+       /// The hash of the [`RawInvoice`] that was signed.
        pub fn signable_hash(&self) -> &[u8; 32] {
                &self.hash
        }
 
-       /// InvoiceSignature for the invoice.
+       /// Signature for the invoice.
        pub fn signature(&self) -> &InvoiceSignature {
                &self.signature
        }
@@ -881,8 +974,8 @@ impl RawInvoice {
                )
        }
 
-       /// Signs the invoice using the supplied `sign_function`. This function MAY fail with an error
-       /// of type `E`. Since the signature of a `SignedRawInvoice` is not required to be valid there
+       /// Signs the invoice using the supplied `sign_method`. This function MAY fail with an error of
+       /// type `E`. Since the signature of a [`SignedRawInvoice`] is not required to be valid there
        /// are no constraints regarding the validity of the produced signature.
        ///
        /// This is not exported to bindings users as we don't currently support passing function pointers into methods
@@ -949,6 +1042,10 @@ impl RawInvoice {
                find_extract!(self.known_tagged_fields(), TaggedField::PaymentSecret(ref x), x)
        }
 
+       pub fn payment_metadata(&self) -> Option<&Vec<u8>> {
+               find_extract!(self.known_tagged_fields(), TaggedField::PaymentMetadata(ref x), x)
+       }
+
        pub fn features(&self) -> Option<&InvoiceFeatures> {
                find_extract!(self.known_tagged_fields(), TaggedField::Features(ref x), x)
        }
@@ -1033,6 +1130,11 @@ impl From<PositiveTimestamp> for SystemTime {
 }
 
 impl Invoice {
+       /// The hash of the [`RawInvoice`] that was signed.
+       pub fn signable_hash(&self) -> [u8; 32] {
+               self.signed_invoice.hash
+       }
+
        /// Transform the `Invoice` into it's unchecked version
        pub fn into_signed_raw(self) -> SignedRawInvoice {
                self.signed_invoice
@@ -1137,7 +1239,7 @@ impl Invoice {
                Ok(())
        }
 
-       /// Constructs an `Invoice` from a `SignedRawInvoice` by checking all its invariants.
+       /// Constructs an `Invoice` from a [`SignedRawInvoice`] by checking all its invariants.
        /// ```
        /// use lightning_invoice::*;
        ///
@@ -1215,6 +1317,11 @@ impl Invoice {
                self.signed_invoice.payment_secret().expect("was checked by constructor")
        }
 
+       /// Get the payment metadata blob if one was included in the invoice
+       pub fn payment_metadata(&self) -> Option<&Vec<u8>> {
+               self.signed_invoice.payment_metadata()
+       }
+
        /// Get the invoice features if they were included in the invoice
        pub fn features(&self) -> Option<&InvoiceFeatures> {
                self.signed_invoice.features()
@@ -1290,6 +1397,25 @@ impl Invoice {
                self.signed_invoice.fallbacks()
        }
 
+       /// Returns a list of all fallback addresses as [`Address`]es
+       pub fn fallback_addresses(&self) -> Vec<Address> {
+               self.fallbacks().iter().map(|fallback| {
+                       let payload = match fallback {
+                               Fallback::SegWitProgram { version, program } => {
+                                       Payload::WitnessProgram { version: *version, program: program.to_vec() }
+                               }
+                               Fallback::PubKeyHash(pkh) => {
+                                       Payload::PubkeyHash(*pkh)
+                               }
+                               Fallback::ScriptHash(sh) => {
+                                       Payload::ScriptHash(*sh)
+                               }
+                       };
+
+                       Address { payload, network: self.network() }
+               }).collect()
+       }
+
        /// Returns a list of all routes included in the invoice
        pub fn private_routes(&self) -> Vec<&PrivateRoute> {
                self.signed_invoice.private_routes()
@@ -1307,6 +1433,13 @@ impl Invoice {
                self.signed_invoice.currency()
        }
 
+       /// Returns the network for which the invoice was issued
+       ///
+       /// This is not exported to bindings users, see [`Self::currency`] instead.
+       pub fn network(&self) -> Network {
+               self.signed_invoice.currency().into()
+       }
+
        /// Returns the amount if specified in the invoice as millisatoshis.
        pub fn amount_milli_satoshis(&self) -> Option<u64> {
                self.signed_invoice.amount_pico_btc().map(|v| v / 10)
@@ -1337,6 +1470,7 @@ impl TaggedField {
                        TaggedField::Fallback(_) => constants::TAG_FALLBACK,
                        TaggedField::PrivateRoute(_) => constants::TAG_PRIVATE_ROUTE,
                        TaggedField::PaymentSecret(_) => constants::TAG_PAYMENT_SECRET,
+                       TaggedField::PaymentMetadata(_) => constants::TAG_PAYMENT_METADATA,
                        TaggedField::Features(_) => constants::TAG_FEATURES,
                };
 
@@ -1347,7 +1481,7 @@ impl TaggedField {
 impl Description {
 
        /// Creates a new `Description` if `description` is at most 1023 __bytes__ long,
-       /// returns `CreationError::DescriptionTooLong` otherwise
+       /// returns [`CreationError::DescriptionTooLong`] otherwise
        ///
        /// Please note that single characters may use more than one byte due to UTF8 encoding.
        pub fn new(description: String) -> Result<Description, CreationError> {
@@ -1358,7 +1492,7 @@ impl Description {
                }
        }
 
-       /// Returns the underlying description `String`
+       /// Returns the underlying description [`String`]
        pub fn into_inner(self) -> String {
                self.0
        }
@@ -1398,7 +1532,7 @@ impl ExpiryTime {
                ExpiryTime(Duration::from_secs(seconds))
        }
 
-       /// Construct an `ExpiryTime` from a `Duration`, dropping the sub-second part.
+       /// Construct an `ExpiryTime` from a [`Duration`], dropping the sub-second part.
        pub fn from_duration(duration: Duration) -> ExpiryTime {
                Self::from_seconds(duration.as_secs())
        }
@@ -1408,7 +1542,7 @@ impl ExpiryTime {
                self.0.as_secs()
        }
 
-       /// Returns a reference to the underlying `Duration` (=expiry time)
+       /// Returns a reference to the underlying [`Duration`] (=expiry time)
        pub fn as_duration(&self) -> &Duration {
                &self.0
        }
@@ -1460,10 +1594,10 @@ impl Deref for SignedRawInvoice {
        }
 }
 
-/// Errors that may occur when constructing a new `RawInvoice` or `Invoice`
+/// Errors that may occur when constructing a new [`RawInvoice`] or [`Invoice`]
 #[derive(Eq, PartialEq, Debug, Clone)]
 pub enum CreationError {
-       /// The supplied description string was longer than 639 __bytes__ (see [`Description::new(…)`](./struct.Description.html#method.new))
+       /// The supplied description string was longer than 639 __bytes__ (see [`Description::new`])
        DescriptionTooLong,
 
        /// The specified route has too many hops and can't be encoded
@@ -1504,7 +1638,7 @@ impl Display for CreationError {
 #[cfg(feature = "std")]
 impl std::error::Error for CreationError { }
 
-/// Errors that may occur when converting a `RawInvoice` to an `Invoice`. They relate to the
+/// Errors that may occur when converting a [`RawInvoice`] to an [`Invoice`]. They relate to the
 /// requirements sections in BOLT #11
 #[derive(Eq, PartialEq, Debug, Clone)]
 pub enum SemanticError {
@@ -1560,7 +1694,7 @@ impl Display for SemanticError {
 #[cfg(feature = "std")]
 impl std::error::Error for SemanticError { }
 
-/// When signing using a fallible method either an user-supplied `SignError` or a `CreationError`
+/// When signing using a fallible method either an user-supplied `SignError` or a [`CreationError`]
 /// may occur.
 #[derive(Eq, PartialEq, Debug, Clone)]
 pub enum SignOrCreationError<S = ()> {
@@ -1591,7 +1725,7 @@ impl<'de> Deserialize<'de> for Invoice {
        fn deserialize<D>(deserializer: D) -> Result<Invoice, D::Error> where D: Deserializer<'de> {
                let bolt11 = String::deserialize(deserializer)?
                        .parse::<Invoice>()
-                       .map_err(|e| D::Error::custom(format!("{:?}", e)))?;
+                       .map_err(|e| D::Error::custom(alloc::format!("{:?}", e)))?;
 
                Ok(bolt11)
        }
@@ -1599,6 +1733,7 @@ impl<'de> Deserialize<'de> for Invoice {
 
 #[cfg(test)]
 mod test {
+       use bitcoin::Script;
        use bitcoin_hashes::hex::FromHex;
        use bitcoin_hashes::sha256;
 
@@ -1962,7 +2097,7 @@ mod test {
                        .payee_pub_key(public_key)
                        .expiry_time(Duration::from_secs(54321))
                        .min_final_cltv_expiry_delta(144)
-                       .fallback(Fallback::PubKeyHash([0;20]))
+                       .fallback(Fallback::PubKeyHash(PubkeyHash::from_slice(&[0;20]).unwrap()))
                        .private_route(route_1.clone())
                        .private_route(route_2.clone())
                        .description_hash(sha256::Hash::from_slice(&[3;32][..]).unwrap())
@@ -1988,7 +2123,9 @@ mod test {
                assert_eq!(invoice.payee_pub_key(), Some(&public_key));
                assert_eq!(invoice.expiry_time(), Duration::from_secs(54321));
                assert_eq!(invoice.min_final_cltv_expiry_delta(), 144);
-               assert_eq!(invoice.fallbacks(), vec![&Fallback::PubKeyHash([0;20])]);
+               assert_eq!(invoice.fallbacks(), vec![&Fallback::PubKeyHash(PubkeyHash::from_slice(&[0;20]).unwrap())]);
+               let address = Address::from_script(&Script::new_p2pkh(&PubkeyHash::from_slice(&[0;20]).unwrap()), Network::Testnet).unwrap();
+               assert_eq!(invoice.fallback_addresses(), vec![address]);
                assert_eq!(invoice.private_routes(), vec![&PrivateRoute(route_1), &PrivateRoute(route_2)]);
                assert_eq!(
                        invoice.description(),
index b1aee9e22b27f45913911adb2cfde4987056d8e9..11757be2e3a2f8399661da3b9b81d51f8457d6a6 100644 (file)
@@ -16,8 +16,8 @@ use bitcoin_hashes::Hash;
 use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use lightning::chain::keysinterface::{NodeSigner, SignerProvider, EntropySource};
-use lightning::ln::{PaymentHash, PaymentSecret};
-use lightning::ln::channelmanager::{ChannelManager, PaymentId, Retry, RetryableSendFailure};
+use lightning::ln::PaymentHash;
+use lightning::ln::channelmanager::{ChannelManager, PaymentId, Retry, RetryableSendFailure, RecipientOnionFields};
 use lightning::routing::router::{PaymentParameters, RouteParameters, Router};
 use lightning::util::logger::Logger;
 
@@ -145,7 +145,10 @@ fn pay_invoice_using_amount<P: Deref>(
        payer: P
 ) -> Result<(), PaymentError> where P::Target: Payer {
        let payment_hash = PaymentHash((*invoice.payment_hash()).into_inner());
-       let payment_secret = Some(*invoice.payment_secret());
+       let recipient_onion = RecipientOnionFields {
+               payment_secret: Some(*invoice.payment_secret()),
+               payment_metadata: invoice.payment_metadata().map(|v| v.clone()),
+       };
        let mut payment_params = PaymentParameters::from_node_id(invoice.recover_payee_pub_key(),
                invoice.min_final_cltv_expiry_delta() as u32)
                .with_expiry_time(expiry_time_from_unix_epoch(invoice).as_secs())
@@ -158,7 +161,7 @@ fn pay_invoice_using_amount<P: Deref>(
                final_value_msat: amount_msats,
        };
 
-       payer.send_payment(payment_hash, &payment_secret, payment_id, route_params, retry_strategy)
+       payer.send_payment(payment_hash, recipient_onion, payment_id, route_params, retry_strategy)
 }
 
 fn expiry_time_from_unix_epoch(invoice: &Invoice) -> Duration {
@@ -182,7 +185,7 @@ trait Payer {
        ///
        /// [`Route`]: lightning::routing::router::Route
        fn send_payment(
-               &self, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry
        ) -> Result<(), PaymentError>;
 }
@@ -199,10 +202,10 @@ where
                L::Target: Logger,
 {
        fn send_payment(
-               &self, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry
        ) -> Result<(), PaymentError> {
-               self.send_payment_with_retry(payment_hash, payment_secret, payment_id, route_params, retry_strategy)
+               self.send_payment(payment_hash, recipient_onion, payment_id, route_params, retry_strategy)
                        .map_err(PaymentError::Sending)
        }
 }
@@ -212,7 +215,9 @@ mod tests {
        use super::*;
        use crate::{InvoiceBuilder, Currency};
        use bitcoin_hashes::sha256::Hash as Sha256;
-       use lightning::ln::PaymentPreimage;
+       use lightning::events::Event;
+       use lightning::ln::msgs::ChannelMessageHandler;
+       use lightning::ln::{PaymentPreimage, PaymentSecret};
        use lightning::ln::functional_test_utils::*;
        use secp256k1::{SecretKey, Secp256k1};
        use std::collections::VecDeque;
@@ -249,7 +254,7 @@ mod tests {
 
        impl Payer for TestPayer {
                fn send_payment(
-                       &self, _payment_hash: PaymentHash, _payment_secret: &Option<PaymentSecret>,
+                       &self, _payment_hash: PaymentHash, _recipient_onion: RecipientOnionFields,
                        _payment_id: PaymentId, route_params: RouteParameters, _retry_strategy: Retry
                ) -> Result<(), PaymentError> {
                        self.check_value_msats(Amount(route_params.final_value_msat));
@@ -349,4 +354,52 @@ mod tests {
                        _ => panic!()
                }
        }
+
+       #[test]
+       #[cfg(feature = "std")]
+       fn payment_metadata_end_to_end() {
+               // Test that a payment metadata read from an invoice passed to `pay_invoice` makes it all
+               // the way out through the `PaymentClaimable` event.
+               let chanmon_cfgs = create_chanmon_cfgs(2);
+               let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+               let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+               let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+               create_announced_chan_between_nodes(&nodes, 0, 1);
+
+               let payment_metadata = vec![42, 43, 44, 45, 46, 47, 48, 49, 42];
+
+               let (payment_hash, payment_secret) =
+                       nodes[1].node.create_inbound_payment(None, 7200, None).unwrap();
+
+               let invoice = InvoiceBuilder::new(Currency::Bitcoin)
+                       .description("test".into())
+                       .payment_hash(Sha256::from_slice(&payment_hash.0).unwrap())
+                       .payment_secret(payment_secret)
+                       .current_timestamp()
+                       .min_final_cltv_expiry_delta(144)
+                       .amount_milli_satoshis(50_000)
+                       .payment_metadata(payment_metadata.clone())
+                       .build_signed(|hash| {
+                               Secp256k1::new().sign_ecdsa_recoverable(hash,
+                                       &nodes[1].keys_manager.backing.get_node_secret_key())
+                       })
+                       .unwrap();
+
+               pay_invoice(&invoice, Retry::Attempts(0), nodes[0].node).unwrap();
+               check_added_monitors(&nodes[0], 1);
+               let send_event = SendEvent::from_node(&nodes[0]);
+               nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
+               commitment_signed_dance!(nodes[1], nodes[0], &send_event.commitment_msg, false);
+
+               expect_pending_htlcs_forwardable!(nodes[1]);
+
+               let mut events = nodes[1].node.get_and_clear_pending_events();
+               assert_eq!(events.len(), 1);
+               match events.pop().unwrap() {
+                       Event::PaymentClaimable { onion_fields, .. } => {
+                               assert_eq!(Some(payment_metadata), onion_fields.unwrap().payment_metadata);
+                       },
+                       _ => panic!("Unexpected event")
+               }
+       }
 }
index c9fca01f11f48fa9923e7d748ec824f43622b7f6..0dca180cab370a79e9358b89d7f1e1d2e51a8c95 100644 (file)
@@ -329,7 +329,7 @@ impl ToBase32 for Fallback {
        fn write_base32<W: WriteBase32>(&self, writer: &mut W) -> Result<(), <W as WriteBase32>::Err> {
                match *self {
                        Fallback::SegWitProgram {version: v, program: ref p} => {
-                               writer.write_u5(v)?;
+                               writer.write_u5(Into::<u5>::into(v))?;
                                p.write_base32(writer)
                        },
                        Fallback::PubKeyHash(ref hash) => {
@@ -446,6 +446,9 @@ impl ToBase32 for TaggedField {
                        TaggedField::PaymentSecret(ref payment_secret) => {
                                  write_tagged_field(writer, constants::TAG_PAYMENT_SECRET, payment_secret)
                        },
+                       TaggedField::PaymentMetadata(ref payment_metadata) => {
+                                 write_tagged_field(writer, constants::TAG_PAYMENT_METADATA, payment_metadata)
+                       },
                        TaggedField::Features(ref features) => {
                                write_tagged_field(writer, constants::TAG_FEATURES, features)
                        },
index fef7a78af145af8d2c8a7a6918c865e8dc06fd76..4f421b9ecc0af095f8d45b95f9ac87c63ad7518c 100644 (file)
@@ -768,13 +768,13 @@ mod test {
        use crate::{Currency, Description, InvoiceDescription, SignOrCreationError, CreationError};
        use bitcoin_hashes::{Hash, sha256};
        use bitcoin_hashes::sha256::Hash as Sha256;
-       use lightning::chain::keysinterface::{EntropySource, PhantomKeysManager};
+       use lightning::chain::keysinterface::PhantomKeysManager;
        use lightning::events::{MessageSendEvent, MessageSendEventsProvider, Event};
        use lightning::ln::{PaymentPreimage, PaymentHash};
-       use lightning::ln::channelmanager::{PhantomRouteHints, MIN_FINAL_CLTV_EXPIRY_DELTA, PaymentId};
+       use lightning::ln::channelmanager::{PhantomRouteHints, MIN_FINAL_CLTV_EXPIRY_DELTA, PaymentId, RecipientOnionFields, Retry};
        use lightning::ln::functional_test_utils::*;
        use lightning::ln::msgs::ChannelMessageHandler;
-       use lightning::routing::router::{PaymentParameters, RouteParameters, find_route};
+       use lightning::routing::router::{PaymentParameters, RouteParameters};
        use lightning::util::test_utils;
        use lightning::util::config::UserConfig;
        use crate::utils::create_invoice_from_channelmanager_and_duration_since_epoch;
@@ -844,20 +844,12 @@ mod test {
                        payment_params,
                        final_value_msat: invoice.amount_milli_satoshis().unwrap(),
                };
-               let first_hops = nodes[0].node.list_usable_channels();
-               let network_graph = &node_cfgs[0].network_graph;
-               let logger = test_utils::TestLogger::new();
-               let scorer = test_utils::TestScorer::new();
-               let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
-               let route = find_route(
-                       &nodes[0].node.get_our_node_id(), &route_params, network_graph,
-                       Some(&first_hops.iter().collect::<Vec<_>>()), &logger, &scorer, &random_seed_bytes
-               ).unwrap();
-
                let payment_event = {
                        let mut payment_hash = PaymentHash([0; 32]);
                        payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]);
-                       nodes[0].node.send_payment(&route, payment_hash, &Some(*invoice.payment_secret()), PaymentId(payment_hash.0)).unwrap();
+                       nodes[0].node.send_payment(payment_hash,
+                               RecipientOnionFields::secret_only(*invoice.payment_secret()),
+                               PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
                        let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                        assert_eq!(added_monitors.len(), 1);
                        added_monitors.clear();
@@ -1308,19 +1300,12 @@ mod test {
                        payment_params,
                        final_value_msat: invoice.amount_milli_satoshis().unwrap(),
                };
-               let first_hops = nodes[0].node.list_usable_channels();
-               let network_graph = &node_cfgs[0].network_graph;
-               let logger = test_utils::TestLogger::new();
-               let scorer = test_utils::TestScorer::new();
-               let random_seed_bytes = chanmon_cfgs[1].keys_manager.get_secure_random_bytes();
-               let route = find_route(
-                       &nodes[0].node.get_our_node_id(), &params, network_graph,
-                       Some(&first_hops.iter().collect::<Vec<_>>()), &logger, &scorer, &random_seed_bytes
-               ).unwrap();
                let (payment_event, fwd_idx) = {
                        let mut payment_hash = PaymentHash([0; 32]);
                        payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]);
-                       nodes[0].node.send_payment(&route, payment_hash, &Some(*invoice.payment_secret()), PaymentId(payment_hash.0)).unwrap();
+                       nodes[0].node.send_payment(payment_hash,
+                               RecipientOnionFields::secret_only(*invoice.payment_secret()),
+                               PaymentId(payment_hash.0), params, Retry::Attempts(0)).unwrap();
                        let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                        assert_eq!(added_monitors.len(), 1);
                        added_monitors.clear();
@@ -1349,7 +1334,7 @@ mod test {
                nodes[fwd_idx].node.process_pending_htlc_forwards();
 
                let payment_preimage_opt = if user_generated_pmt_hash { None } else { Some(payment_preimage) };
-               expect_payment_claimable!(&nodes[fwd_idx], payment_hash, payment_secret, payment_amt, payment_preimage_opt, route.paths[0].last().unwrap().pubkey);
+               expect_payment_claimable!(&nodes[fwd_idx], payment_hash, payment_secret, payment_amt, payment_preimage_opt, invoice.recover_payee_pub_key());
                do_claim_payment_along_route(&nodes[0], &[&vec!(&nodes[fwd_idx])[..]], false, payment_preimage);
                let events = nodes[0].node.get_and_clear_pending_events();
                assert_eq!(events.len(), 2);
index 272d9062a6e4c9a062e622c55f2ed568bbae5409..ef5a4d32b30a0caa7c397217fb808a64ca2be877 100644 (file)
@@ -5,9 +5,10 @@ extern crate lightning_invoice;
 extern crate secp256k1;
 extern crate hex;
 
+use bitcoin::util::address::WitnessVersion;
+use bitcoin::{PubkeyHash, ScriptHash};
 use bitcoin_hashes::hex::FromHex;
 use bitcoin_hashes::{sha256, Hash};
-use bech32::u5;
 use lightning::ln::PaymentSecret;
 use lightning::routing::gossip::RoutingFees;
 use lightning::routing::router::{RouteHint, RouteHintHop};
@@ -115,7 +116,7 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                                .payment_hash(sha256::Hash::from_hex(
                                        "0001020304050607080900010203040506070809000102030405060708090102"
                                ).unwrap())
-                               .fallback(Fallback::PubKeyHash([49, 114, 181, 101, 79, 102, 131, 200, 251, 20, 105, 89, 211, 71, 206, 48, 60, 174, 76, 167]))
+                               .fallback(Fallback::PubKeyHash(PubkeyHash::from_slice(&[49, 114, 181, 101, 79, 102, 131, 200, 251, 20, 105, 89, 211, 71, 206, 48, 60, 174, 76, 167]).unwrap()))
                                .build_raw()
                                .unwrap()
                                .sign(|_| {
@@ -137,7 +138,7 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                                .payment_hash(sha256::Hash::from_hex(
                                        "0001020304050607080900010203040506070809000102030405060708090102"
                                ).unwrap())
-                               .fallback(Fallback::PubKeyHash([4, 182, 31, 125, 193, 234, 13, 201, 148, 36, 70, 76, 196, 6, 77, 197, 100, 217, 30, 137]))
+                               .fallback(Fallback::PubKeyHash(PubkeyHash::from_slice(&[4, 182, 31, 125, 193, 234, 13, 201, 148, 36, 70, 76, 196, 6, 77, 197, 100, 217, 30, 137]).unwrap()))
                                .private_route(RouteHint(vec![RouteHintHop {
                                        src_node_id: PublicKey::from_slice(&hex::decode(
                                                        "029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255"
@@ -176,7 +177,7 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                                .payment_hash(sha256::Hash::from_hex(
                                        "0001020304050607080900010203040506070809000102030405060708090102"
                                ).unwrap())
-                               .fallback(Fallback::ScriptHash([143, 85, 86, 59, 154, 25, 243, 33, 194, 17, 233, 185, 243, 140, 223, 104, 110, 160, 120, 69]))
+                               .fallback(Fallback::ScriptHash(ScriptHash::from_slice(&[143, 85, 86, 59, 154, 25, 243, 33, 194, 17, 233, 185, 243, 140, 223, 104, 110, 160, 120, 69]).unwrap()))
                                .build_raw()
                                .unwrap()
                                .sign(|_| {
@@ -198,7 +199,7 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                                .payment_hash(sha256::Hash::from_hex(
                                        "0001020304050607080900010203040506070809000102030405060708090102"
                                ).unwrap())
-                               .fallback(Fallback::SegWitProgram { version: u5::try_from_u8(0).unwrap(),
+                               .fallback(Fallback::SegWitProgram { version: WitnessVersion::V0,
                                        program: vec![117, 30, 118, 232, 25, 145, 150, 212, 84, 148, 28, 69, 209, 179, 163, 35, 241, 67, 59, 214]
                                })
                                .build_raw()
@@ -222,7 +223,7 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                                .payment_hash(sha256::Hash::from_hex(
                                        "0001020304050607080900010203040506070809000102030405060708090102"
                                ).unwrap())
-                               .fallback(Fallback::SegWitProgram { version: u5::try_from_u8(0).unwrap(),
+                               .fallback(Fallback::SegWitProgram { version: WitnessVersion::V0,
                                        program: vec![24, 99, 20, 60, 20, 197, 22, 104, 4, 189, 25, 32, 51, 86, 218, 19, 108, 152, 86, 120, 205, 77, 39, 161, 184, 198, 50, 150, 4, 144, 50, 98]
                                })
                                .build_raw()
@@ -331,6 +332,56 @@ fn get_test_tuples() -> Vec<(String, SignedRawInvoice, bool, bool)> {
                        true, // Different features than set in InvoiceBuilder
                        true, // Some unknown fields
                ),
+               ( // Older version of the payment metadata test with a payment_pubkey set
+                       "lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqnp4q0n326hr8v9zprg8gsvezcch06gfaqqhde2aj730yg0durunfhv66sp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgqy9gw6ymamd20jumvdgpfphkhp8fzhhdhycw36egcmla5vlrtrmhs9t7psfy3hkkdqzm9eq64fjg558znccds5nhsfmxveha5xe0dykgpspdha0".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_milli_satoshis(1_000_000_000)
+                               .duration_since_epoch(Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("payment metadata inside".to_owned())
+                               .payment_metadata(hex::decode("01fafaf0").unwrap())
+                               .require_payment_metadata()
+                               .payee_pub_key(PublicKey::from_slice(&hex::decode(
+                                               "03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad"
+                                       ).unwrap()).unwrap())
+                               .payment_secret(PaymentSecret([0x11; 32]))
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               &hex::decode("2150ed137ddb54f9736c6a0290ded709d22bddb7261d1d6518dffb467c6b1eef02afc182491bdacd00b65c83554c914a1c53c61b0a4ef04eccccdfb4365ed259").unwrap(),
+                                               RecoveryId::from_i32(1).unwrap()
+                                       )
+                               }).unwrap(),
+                       false, // Different features than set in InvoiceBuilder
+                       true, // Some unknown fields
+               ),
+               (
+                       "lnbc10m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdp9wpshjmt9de6zqmt9w3skgct5vysxjmnnd9jx2mq8q8a04uqsp5zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs9q2gqqqqqqsgq7hf8he7ecf7n4ffphs6awl9t6676rrclv9ckg3d3ncn7fct63p6s365duk5wrk202cfy3aj5xnnp5gs3vrdvruverwwq7yzhkf5a3xqpd05wjc".to_owned(),
+                       InvoiceBuilder::new(Currency::Bitcoin)
+                               .amount_milli_satoshis(1_000_000_000)
+                               .duration_since_epoch(Duration::from_secs(1496314658))
+                               .payment_hash(sha256::Hash::from_hex(
+                                       "0001020304050607080900010203040506070809000102030405060708090102"
+                               ).unwrap())
+                               .description("payment metadata inside".to_owned())
+                               .payment_metadata(hex::decode("01fafaf0").unwrap())
+                               .require_payment_metadata()
+                               .payment_secret(PaymentSecret([0x11; 32]))
+                               .build_raw()
+                               .unwrap()
+                               .sign(|_| {
+                                       RecoverableSignature::from_compact(
+                                               &hex::decode("f5d27be7d9c27d3aa521bc35d77cabd6bda18f1f61716445b19e27e4e17a887508ea8de5a8e1d94f561248f65434e61a221160dac1f1991b9c0f1057b269d898").unwrap(),
+                                               RecoveryId::from_i32(1).unwrap()
+                                       )
+                               }).unwrap(),
+                       false, // Different features than set in InvoiceBuilder
+                       true, // Some unknown fields
+               ),
+
        ]
 }
 
index 6250628d24ae8786971ffba6b46e7d3b093a71ad..a0fdcf8399d8865134c00ced502d0bc476fe3575 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-net-tokio"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning/"
@@ -16,9 +16,9 @@ rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.114", path = "../lightning" }
+lightning = { version = "0.0.115", path = "../lightning" }
 tokio = { version = "1.0", features = [ "io-util", "macros", "rt", "sync", "net", "time" ] }
 
 [dev-dependencies]
 tokio = { version = "1.14", features = [ "io-util", "macros", "rt", "rt-multi-thread", "sync", "net", "time" ] }
-lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }
index aeb5c5b7a87280c68720975c69c5d6167f5fab1b..48f1736d0c2296988c5d0f7c2c110421d5ca319f 100644 (file)
@@ -8,64 +8,19 @@
 // licenses.
 
 //! A socket handling library for those running in Tokio environments who wish to use
-//! rust-lightning with native TcpStreams.
+//! rust-lightning with native [`TcpStream`]s.
 //!
 //! Designed to be as simple as possible, the high-level usage is almost as simple as "hand over a
-//! TcpStream and a reference to a PeerManager and the rest is handled", except for the
-//! [Event](../lightning/util/events/enum.Event.html) handling mechanism; see example below.
+//! [`TcpStream`] and a reference to a [`PeerManager`] and the rest is handled".
 //!
-//! The PeerHandler, due to the fire-and-forget nature of this logic, must be an Arc, and must use
-//! the SocketDescriptor provided here as the PeerHandler's SocketDescriptor.
+//! The [`PeerManager`], due to the fire-and-forget nature of this logic, must be a reference,
+//! (e.g. an [`Arc`]) and must use the [`SocketDescriptor`] provided here as the [`PeerManager`]'s
+//! `SocketDescriptor` implementation.
 //!
-//! Three methods are exposed to register a new connection for handling in tokio::spawn calls; see
-//! their individual docs for details.
+//! Three methods are exposed to register a new connection for handling in [`tokio::spawn`] calls;
+//! see their individual docs for details.
 //!
-//! # Example
-//! ```
-//! use std::net::TcpStream;
-//! use bitcoin::secp256k1::PublicKey;
-//! use lightning::events::{Event, EventHandler, EventsProvider};
-//! use std::net::SocketAddr;
-//! use std::sync::Arc;
-//!
-//! // Define concrete types for our high-level objects:
-//! type TxBroadcaster = dyn lightning::chain::chaininterface::BroadcasterInterface + Send + Sync;
-//! type FeeEstimator = dyn lightning::chain::chaininterface::FeeEstimator + Send + Sync;
-//! type Logger = dyn lightning::util::logger::Logger + Send + Sync;
-//! type NodeSigner = dyn lightning::chain::keysinterface::NodeSigner + Send + Sync;
-//! type UtxoLookup = dyn lightning::routing::utxo::UtxoLookup + Send + Sync;
-//! type ChainFilter = dyn lightning::chain::Filter + Send + Sync;
-//! type DataPersister = dyn lightning::chain::chainmonitor::Persist<lightning::chain::keysinterface::InMemorySigner> + Send + Sync;
-//! type ChainMonitor = lightning::chain::chainmonitor::ChainMonitor<lightning::chain::keysinterface::InMemorySigner, Arc<ChainFilter>, Arc<TxBroadcaster>, Arc<FeeEstimator>, Arc<Logger>, Arc<DataPersister>>;
-//! type ChannelManager = Arc<lightning::ln::channelmanager::SimpleArcChannelManager<ChainMonitor, TxBroadcaster, FeeEstimator, Logger>>;
-//! type PeerManager = Arc<lightning::ln::peer_handler::SimpleArcPeerManager<lightning_net_tokio::SocketDescriptor, ChainMonitor, TxBroadcaster, FeeEstimator, UtxoLookup, Logger>>;
-//!
-//! // Connect to node with pubkey their_node_id at addr:
-//! async fn connect_to_node(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, their_node_id: PublicKey, addr: SocketAddr) {
-//!    lightning_net_tokio::connect_outbound(peer_manager, their_node_id, addr).await;
-//!    loop {
-//!            let event_handler = |event: Event| {
-//!                    // Handle the event!
-//!            };
-//!            channel_manager.await_persistable_update();
-//!            channel_manager.process_pending_events(&event_handler);
-//!            chain_monitor.process_pending_events(&event_handler);
-//!    }
-//! }
-//!
-//! // Begin reading from a newly accepted socket and talk to the peer:
-//! async fn accept_socket(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, socket: TcpStream) {
-//!    lightning_net_tokio::setup_inbound(peer_manager, socket);
-//!    loop {
-//!            let event_handler = |event: Event| {
-//!                    // Handle the event!
-//!            };
-//!            channel_manager.await_persistable_update();
-//!            channel_manager.process_pending_events(&event_handler);
-//!            chain_monitor.process_pending_events(&event_handler);
-//!    }
-//! }
-//! ```
+//! [`PeerManager`]: lightning::ln::peer_handler::PeerManager
 
 // Prefix these with `rustdoc::` when we update our MSRV to be >= 1.52 to remove warnings.
 #![deny(broken_intra_doc_links)]
@@ -234,7 +189,7 @@ impl Connection {
                        // our timeslice to another task we may just spin on this peer, starving other peers
                        // and eventually disconnecting them for ping timeouts. Instead, we explicitly yield
                        // here.
-                       tokio::task::yield_now().await;
+                       let _ = tokio::task::yield_now().await;
                };
                let writer_option = us.lock().unwrap().writer.take();
                if let Some(mut writer) = writer_option {
index 01415f29f205b7e4121a485948978023fe652df9..88132bdcfd660ed70e367ed63c420a08244e408c 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-persister"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Valentine Wallace", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning/"
@@ -18,11 +18,11 @@ _bench_unstable = ["lightning/_bench_unstable"]
 
 [dependencies]
 bitcoin = "0.29.0"
-lightning = { version = "0.0.114", path = "../lightning" }
+lightning = { version = "0.0.115", path = "../lightning" }
 libc = "0.2"
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["winbase"] }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }
index 1f0f6f038007b905e5e8d30d66aebe4a1ce645b7..6673431d93b210d24c87ac42e9c9561b4728d66b 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-rapid-gossip-sync"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Arik Sosman <git@arik.io>"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning"
@@ -16,8 +16,8 @@ std = ["lightning/std"]
 _bench_unstable = []
 
 [dependencies]
-lightning = { version = "0.0.114", path = "../lightning", default-features = false }
+lightning = { version = "0.0.115", path = "../lightning", default-features = false }
 bitcoin = { version = "0.29.0", default-features = false }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["_test_utils"] }
index da59b37b10a673c9420b9d2232c81758f0811f59..c8f140cc18955d8225bbf300fcda2aee6ddae267 100644 (file)
 //! # use lightning::util::logger::{Logger, Record};
 //! # struct FakeLogger {}
 //! # impl Logger for FakeLogger {
-//! #     fn log(&self, record: &Record) { unimplemented!() }
+//! #     fn log(&self, record: &Record) { }
 //! # }
 //! # let logger = FakeLogger {};
 //!
 //! let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
 //! let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
 //! let snapshot_contents: &[u8] = &[0; 0];
-//! let new_last_sync_timestamp_result = rapid_sync.update_network_graph(snapshot_contents);
+//! // In no-std you need to provide the current time in unix epoch seconds
+//! // otherwise you can use update_network_graph
+//! let current_time_unix = 0;
+//! let new_last_sync_timestamp_result = rapid_sync.update_network_graph_no_std(snapshot_contents, Some(current_time_unix));
 //! ```
 
 #![cfg_attr(all(not(feature = "std"), not(test)), no_std)]
@@ -128,6 +131,7 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
        /// Returns the last sync timestamp to be used the next time rapid sync data is queried.
        ///
        /// `update_data`: `&[u8]` binary stream that comprises the update data
+       #[cfg(feature = "std")]
        pub fn update_network_graph(&self, update_data: &[u8]) -> Result<u32, GraphSyncError> {
                let mut read_cursor = io::Cursor::new(update_data);
                self.update_network_graph_from_byte_stream(&mut read_cursor)
index 8d36dfe38844f75d189e4c27896b3f3a4d3e4891..b387d49998e9a38d6f1693e1a18d05c4da57e675 100644 (file)
@@ -10,7 +10,7 @@ use lightning::ln::msgs::{
 };
 use lightning::routing::gossip::NetworkGraph;
 use lightning::util::logger::Logger;
-use lightning::{log_warn, log_trace, log_given_level};
+use lightning::{log_debug, log_warn, log_trace, log_given_level, log_gossip};
 use lightning::util::ser::{BigSize, Readable};
 use lightning::io;
 
@@ -38,13 +38,14 @@ const MAX_INITIAL_NODE_ID_VECTOR_CAPACITY: u32 = 50_000;
 const STALE_RGS_UPDATE_AGE_LIMIT_SECS: u64 = 60 * 60 * 24 * 14;
 
 impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L::Target: Logger {
+       #[cfg(feature = "std")]
        pub(crate) fn update_network_graph_from_byte_stream<R: io::Read>(
                &self,
                read_cursor: &mut R,
        ) -> Result<u32, GraphSyncError> {
                #[allow(unused_mut, unused_assignments)]
                let mut current_time_unix = None;
-               #[cfg(all(feature = "std", not(test)))]
+               #[cfg(not(test))]
                {
                        // Note that many tests rely on being able to set arbitrarily old timestamps, thus we
                        // disable this check during tests!
@@ -58,6 +59,7 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                mut read_cursor: &mut R,
                current_time_unix: Option<u64>
        ) -> Result<u32, GraphSyncError> {
+               log_trace!(self.logger, "Processing RGS data...");
                let mut prefix = [0u8; 4];
                read_cursor.read_exact(&mut prefix)?;
 
@@ -110,6 +112,9 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                        let node_id_1 = node_ids[node_id_1_index.0 as usize];
                        let node_id_2 = node_ids[node_id_2_index.0 as usize];
 
+                       log_gossip!(self.logger, "Adding channel {} from RGS announcement at {}",
+                               short_channel_id, latest_seen_timestamp);
+
                        let announcement_result = network_graph.add_channel_from_partial_announcement(
                                short_channel_id,
                                backdated_timestamp as u64,
@@ -130,6 +135,8 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                previous_scid = 0; // updates start at a new scid
 
                let update_count: u32 = Readable::read(read_cursor)?;
+               log_debug!(self.logger, "Processing RGS update from {} with {} nodes, {} channel announcements and {} channel updates.",
+                       latest_seen_timestamp, node_id_count, announcement_count, update_count);
                if update_count == 0 {
                        return Ok(latest_seen_timestamp);
                }
@@ -217,6 +224,8 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                                continue;
                        }
 
+                       log_gossip!(self.logger, "Updating channel {} with flags {} from RGS announcement at {}",
+                               short_channel_id, channel_flags, latest_seen_timestamp);
                        match network_graph.update_channel_unsigned(&synthetic_update) {
                                Ok(_) => {},
                                Err(LightningError { action: ErrorAction::IgnoreDuplicateGossip, .. }) => {},
@@ -229,7 +238,13 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                }
 
                self.network_graph.set_last_rapid_gossip_sync_timestamp(latest_seen_timestamp);
+
+               if let Some(time) = current_time_unix {
+                       self.network_graph.remove_stale_channels_and_tracking_with_time(time)
+               }
+
                self.is_initial_sync_complete.store(true, Ordering::Release);
+               log_trace!(self.logger, "Done processing RGS data from {}", latest_seen_timestamp);
                Ok(latest_seen_timestamp)
        }
 }
@@ -238,7 +253,9 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
 mod tests {
        use bitcoin::Network;
 
+       #[cfg(feature = "std")]
        use lightning::ln::msgs::DecodeError;
+
        use lightning::routing::gossip::NetworkGraph;
        use lightning::util::test_utils::TestLogger;
 
@@ -266,6 +283,7 @@ mod tests {
        const VALID_BINARY_TIMESTAMP: u64 = 1642291930;
 
        #[test]
+       #[cfg(feature = "std")]
        fn network_graph_fails_to_update_from_clipped_input() {
                let logger = TestLogger::new();
                let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
@@ -297,6 +315,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn incremental_only_update_ignores_missing_channel() {
                let incremental_update_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
@@ -317,6 +336,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn incremental_only_update_fails_without_prior_updates() {
                let announced_update_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
@@ -344,6 +364,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn incremental_only_update_fails_without_prior_same_direction_updates() {
                let initialization_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
@@ -399,6 +420,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn incremental_update_succeeds_with_prior_announcements_and_full_updates() {
                let initialization_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
@@ -458,6 +480,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn update_succeeds_when_duplicate_gossip_is_applied() {
                let initialization_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
@@ -501,6 +524,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn full_update_succeeds() {
                let logger = TestLogger::new();
                let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
@@ -545,6 +569,34 @@ mod tests {
                assert_eq!(network_graph.read_only().channels().len(), 2);
        }
 
+       #[test]
+       fn prunes_after_update() {
+               // this is the timestamp encoded in the binary data of valid_input below
+               let logger = TestLogger::new();
+
+               let latest_nonpruning_time = VALID_BINARY_TIMESTAMP + 60 * 60 * 24 * 7;
+
+               {
+                       let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
+                       assert_eq!(network_graph.read_only().channels().len(), 0);
+
+                       let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
+                       let update_result = rapid_sync.update_network_graph_no_std(&VALID_RGS_BINARY, Some(latest_nonpruning_time));
+                       assert!(update_result.is_ok());
+                       assert_eq!(network_graph.read_only().channels().len(), 2);
+               }
+
+               {
+                       let network_graph = NetworkGraph::new(Network::Bitcoin, &logger);
+                       assert_eq!(network_graph.read_only().channels().len(), 0);
+
+                       let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
+                       let update_result = rapid_sync.update_network_graph_no_std(&VALID_RGS_BINARY, Some(latest_nonpruning_time + 1));
+                       assert!(update_result.is_ok());
+                       assert_eq!(network_graph.read_only().channels().len(), 0);
+               }
+       }
+
        #[test]
        fn timestamp_edge_cases_are_handled_correctly() {
                // this is the timestamp encoded in the binary data of valid_input below
@@ -560,7 +612,7 @@ mod tests {
                        let rapid_sync = RapidGossipSync::new(&network_graph, &logger);
                        let update_result = rapid_sync.update_network_graph_no_std(&VALID_RGS_BINARY, Some(latest_succeeding_time));
                        assert!(update_result.is_ok());
-                       assert_eq!(network_graph.read_only().channels().len(), 2);
+                       assert_eq!(network_graph.read_only().channels().len(), 0);
                }
 
                {
@@ -582,6 +634,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        pub fn update_fails_with_unknown_version() {
                let unknown_version_input = vec![
                        76, 68, 75, 2, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
index 84bccabadb647ffa0a6815ef2965e179ef4c10df..2e9e13296b6c993e1622c84370e20a2cd97e88b0 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-transaction-sync"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Elias Rohrer"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/lightningdevkit/rust-lightning"
@@ -21,15 +21,15 @@ esplora-blocking = ["esplora-client/blocking"]
 async-interface = []
 
 [dependencies]
-lightning = { version = "0.0.114", path = "../lightning", default-features = false }
+lightning = { version = "0.0.115", path = "../lightning", default-features = false }
 bitcoin = { version = "0.29.0", default-features = false }
 bdk-macros = "0.6"
 futures = { version = "0.3", optional = true }
-esplora-client = { version = "0.3", default-features = false, optional = true }
+esplora-client = { version = "0.4", default-features = false, optional = true }
 reqwest = { version = "0.11", optional = true, default-features = false, features = ["json"] }
 
 [dev-dependencies]
-lightning = { version = "0.0.114", path = "../lightning", features = ["std"] }
+lightning = { version = "0.0.115", path = "../lightning", features = ["std"] }
 electrsd = { version = "0.22.0", features = ["legacy", "esplora_a33e97e1", "bitcoind_23_0"] }
 electrum-client = "0.12.0"
 tokio = { version = "1.14.0", features = ["full"] }
index 3f4f5d43503baa409de88300dc93725f2898f4ab..1804be0f1bb50ac26b84c6472a7c1137f5b8ebf8 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning"
-version = "0.0.114"
+version = "0.0.115"
 authors = ["Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/lightningdevkit/rust-lightning/"
@@ -29,6 +29,8 @@ max_level_trace = []
 # This is unsafe to use in production because it may result in the counterparty publishing taking our funds.
 unsafe_revoked_tx_signing = []
 _bench_unstable = []
+# Override signing to not include randomness when generating signatures for test vectors.
+_test_vectors = []
 
 no-std = ["hashbrown", "bitcoin/no-std", "core2/alloc"]
 std = ["bitcoin/std"]
@@ -56,3 +58,6 @@ regex = "1.5.6"
 version = "0.29.0"
 default-features = false
 features = ["bitcoinconsensus", "secp-recovery"]
+
+[target.'cfg(taproot)'.dependencies]
+musig2 = { git = "https://github.com/arik-so/rust-musig2", rev = "27797d7" }
diff --git a/lightning/src/blinded_path/mod.rs b/lightning/src/blinded_path/mod.rs
new file mode 100644 (file)
index 0000000..2cd03b8
--- /dev/null
@@ -0,0 +1,233 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Creating blinded paths and related utilities live here.
+
+pub(crate) mod utils;
+
+use bitcoin::hashes::{Hash, HashEngine};
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey};
+
+use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient};
+use crate::onion_message::ControlTlvs;
+use crate::ln::msgs::DecodeError;
+use crate::ln::onion_utils;
+use crate::util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter};
+use crate::util::ser::{FixedLengthReader, LengthReadableArgs, Readable, VecWriter, Writeable, Writer};
+
+use core::mem;
+use core::ops::Deref;
+use crate::io::{self, Cursor};
+use crate::prelude::*;
+
+/// Onion messages and payments can be sent and received to blinded paths, which serve to hide the
+/// identity of the recipient.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct BlindedPath {
+       /// To send to a blinded path, the sender first finds a route to the unblinded
+       /// `introduction_node_id`, which can unblind its [`encrypted_payload`] to find out the onion
+       /// message or payment's next hop and forward it along.
+       ///
+       /// [`encrypted_payload`]: BlindedHop::encrypted_payload
+       pub(crate) introduction_node_id: PublicKey,
+       /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion
+       /// message or payment.
+       ///
+       /// [`encrypted_payload`]: BlindedHop::encrypted_payload
+       pub(crate) blinding_point: PublicKey,
+       /// The hops composing the blinded path.
+       pub(crate) blinded_hops: Vec<BlindedHop>,
+}
+
+/// Used to construct the blinded hops portion of a blinded path. These hops cannot be identified
+/// by outside observers and thus can be used to hide the identity of the recipient.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct BlindedHop {
+       /// The blinded node id of this hop in a blinded path.
+       pub(crate) blinded_node_id: PublicKey,
+       /// The encrypted payload intended for this hop in a blinded path.
+       // The node sending to this blinded path will later encode this payload into the onion packet for
+       // this hop.
+       pub(crate) encrypted_payload: Vec<u8>,
+}
+
+impl BlindedPath {
+       /// Create a blinded path for an onion message, to be forwarded along `node_pks`. The last node
+       /// pubkey in `node_pks` will be the destination node.
+       ///
+       /// Errors if less than two hops are provided or if `node_pk`(s) are invalid.
+       //  TODO: make all payloads the same size with padding + add dummy hops
+       pub fn new_for_message<ES: EntropySource, T: secp256k1::Signing + secp256k1::Verification>
+               (node_pks: &[PublicKey], entropy_source: &ES, secp_ctx: &Secp256k1<T>) -> Result<Self, ()>
+       {
+               if node_pks.len() < 2 { return Err(()) }
+               let blinding_secret_bytes = entropy_source.get_secure_random_bytes();
+               let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted");
+               let introduction_node_id = node_pks[0];
+
+               Ok(BlindedPath {
+                       introduction_node_id,
+                       blinding_point: PublicKey::from_secret_key(secp_ctx, &blinding_secret),
+                       blinded_hops: blinded_message_hops(secp_ctx, node_pks, &blinding_secret).map_err(|_| ())?,
+               })
+       }
+
+       // Advance the blinded onion message path by one hop, so make the second hop into the new
+       // introduction node.
+       pub(super) fn advance_message_path_by_one<NS: Deref, T: secp256k1::Signing + secp256k1::Verification>
+               (&mut self, node_signer: &NS, secp_ctx: &Secp256k1<T>) -> Result<(), ()>
+               where NS::Target: NodeSigner
+       {
+               let control_tlvs_ss = node_signer.ecdh(Recipient::Node, &self.blinding_point, None)?;
+               let rho = onion_utils::gen_rho_from_shared_secret(&control_tlvs_ss.secret_bytes());
+               let encrypted_control_tlvs = self.blinded_hops.remove(0).encrypted_payload;
+               let mut s = Cursor::new(&encrypted_control_tlvs);
+               let mut reader = FixedLengthReader::new(&mut s, encrypted_control_tlvs.len() as u64);
+               match ChaChaPolyReadAdapter::read(&mut reader, rho) {
+                       Ok(ChaChaPolyReadAdapter { readable: ControlTlvs::Forward(ForwardTlvs {
+                               mut next_node_id, next_blinding_override,
+                       })}) => {
+                               let mut new_blinding_point = match next_blinding_override {
+                                       Some(blinding_point) => blinding_point,
+                                       None => {
+                                               let blinding_factor = {
+                                                       let mut sha = Sha256::engine();
+                                                       sha.input(&self.blinding_point.serialize()[..]);
+                                                       sha.input(control_tlvs_ss.as_ref());
+                                                       Sha256::from_engine(sha).into_inner()
+                                               };
+                                               self.blinding_point.mul_tweak(secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap())
+                                                       .map_err(|_| ())?
+                                       }
+                               };
+                               mem::swap(&mut self.blinding_point, &mut new_blinding_point);
+                               mem::swap(&mut self.introduction_node_id, &mut next_node_id);
+                               Ok(())
+                       },
+                       _ => Err(())
+               }
+       }
+}
+
+/// Construct blinded onion message hops for the given `unblinded_path`.
+fn blinded_message_hops<T: secp256k1::Signing + secp256k1::Verification>(
+       secp_ctx: &Secp256k1<T>, unblinded_path: &[PublicKey], session_priv: &SecretKey
+) -> Result<Vec<BlindedHop>, secp256k1::Error> {
+       let mut blinded_hops = Vec::with_capacity(unblinded_path.len());
+
+       let mut prev_ss_and_blinded_node_id = None;
+       utils::construct_keys_callback(secp_ctx, unblinded_path, None, session_priv, |blinded_node_id, _, _, encrypted_payload_ss, unblinded_pk, _| {
+               if let Some((prev_ss, prev_blinded_node_id)) = prev_ss_and_blinded_node_id {
+                       if let Some(pk) = unblinded_pk {
+                               let payload = ForwardTlvs {
+                                       next_node_id: pk,
+                                       next_blinding_override: None,
+                               };
+                               blinded_hops.push(BlindedHop {
+                                       blinded_node_id: prev_blinded_node_id,
+                                       encrypted_payload: encrypt_payload(payload, prev_ss),
+                               });
+                       } else { debug_assert!(false); }
+               }
+               prev_ss_and_blinded_node_id = Some((encrypted_payload_ss, blinded_node_id));
+       })?;
+
+       if let Some((final_ss, final_blinded_node_id)) = prev_ss_and_blinded_node_id {
+               let final_payload = ReceiveTlvs { path_id: None };
+               blinded_hops.push(BlindedHop {
+                       blinded_node_id: final_blinded_node_id,
+                       encrypted_payload: encrypt_payload(final_payload, final_ss),
+               });
+       } else { debug_assert!(false) }
+
+       Ok(blinded_hops)
+}
+
+/// Encrypt TLV payload to be used as a [`BlindedHop::encrypted_payload`].
+fn encrypt_payload<P: Writeable>(payload: P, encrypted_tlvs_ss: [u8; 32]) -> Vec<u8> {
+       let mut writer = VecWriter(Vec::new());
+       let write_adapter = ChaChaPolyWriteAdapter::new(encrypted_tlvs_ss, &payload);
+       write_adapter.write(&mut writer).expect("In-memory writes cannot fail");
+       writer.0
+}
+
+impl Writeable for BlindedPath {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.introduction_node_id.write(w)?;
+               self.blinding_point.write(w)?;
+               (self.blinded_hops.len() as u8).write(w)?;
+               for hop in &self.blinded_hops {
+                       hop.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl Readable for BlindedPath {
+       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let introduction_node_id = Readable::read(r)?;
+               let blinding_point = Readable::read(r)?;
+               let num_hops: u8 = Readable::read(r)?;
+               if num_hops == 0 { return Err(DecodeError::InvalidValue) }
+               let mut blinded_hops: Vec<BlindedHop> = Vec::with_capacity(num_hops.into());
+               for _ in 0..num_hops {
+                       blinded_hops.push(Readable::read(r)?);
+               }
+               Ok(BlindedPath {
+                       introduction_node_id,
+                       blinding_point,
+                       blinded_hops,
+               })
+       }
+}
+
+impl_writeable!(BlindedHop, {
+       blinded_node_id,
+       encrypted_payload
+});
+
+/// TLVs to encode in an intermediate onion message packet's hop data. When provided in a blinded
+/// route, they are encoded into [`BlindedHop::encrypted_payload`].
+pub(crate) struct ForwardTlvs {
+       /// The node id of the next hop in the onion message's path.
+       pub(super) next_node_id: PublicKey,
+       /// Senders to a blinded path use this value to concatenate the route they find to the
+       /// introduction node with the blinded path.
+       pub(super) next_blinding_override: Option<PublicKey>,
+}
+
+/// Similar to [`ForwardTlvs`], but these TLVs are for the final node.
+pub(crate) struct ReceiveTlvs {
+       /// If `path_id` is `Some`, it is used to identify the blinded path that this onion message is
+       /// sending to. This is useful for receivers to check that said blinded path is being used in
+       /// the right context.
+       pub(super) path_id: Option<[u8; 32]>,
+}
+
+impl Writeable for ForwardTlvs {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               // TODO: write padding
+               encode_tlv_stream!(writer, {
+                       (4, self.next_node_id, required),
+                       (8, self.next_blinding_override, option)
+               });
+               Ok(())
+       }
+}
+
+impl Writeable for ReceiveTlvs {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               // TODO: write padding
+               encode_tlv_stream!(writer, {
+                       (6, self.path_id, option),
+               });
+               Ok(())
+       }
+}
diff --git a/lightning/src/blinded_path/utils.rs b/lightning/src/blinded_path/utils.rs
new file mode 100644 (file)
index 0000000..1993ad9
--- /dev/null
@@ -0,0 +1,98 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Onion message utility methods live here.
+
+use bitcoin::hashes::{Hash, HashEngine};
+use bitcoin::hashes::hmac::{Hmac, HmacEngine};
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey, Scalar};
+use bitcoin::secp256k1::ecdh::SharedSecret;
+
+use super::BlindedPath;
+use crate::ln::onion_utils;
+use crate::onion_message::Destination;
+
+use crate::prelude::*;
+
+// TODO: DRY with onion_utils::construct_onion_keys_callback
+#[inline]
+pub(crate) fn construct_keys_callback<T: secp256k1::Signing + secp256k1::Verification,
+       FType: FnMut(PublicKey, SharedSecret, PublicKey, [u8; 32], Option<PublicKey>, Option<Vec<u8>>)>(
+       secp_ctx: &Secp256k1<T>, unblinded_path: &[PublicKey], destination: Option<Destination>,
+       session_priv: &SecretKey, mut callback: FType
+) -> Result<(), secp256k1::Error> {
+       let mut msg_blinding_point_priv = session_priv.clone();
+       let mut msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv);
+       let mut onion_packet_pubkey_priv = msg_blinding_point_priv.clone();
+       let mut onion_packet_pubkey = msg_blinding_point.clone();
+
+       macro_rules! build_keys {
+               ($pk: expr, $blinded: expr, $encrypted_payload: expr) => {{
+                       let encrypted_data_ss = SharedSecret::new(&$pk, &msg_blinding_point_priv);
+
+                       let blinded_hop_pk = if $blinded { $pk } else {
+                               let hop_pk_blinding_factor = {
+                                       let mut hmac = HmacEngine::<Sha256>::new(b"blinded_node_id");
+                                       hmac.input(encrypted_data_ss.as_ref());
+                                       Hmac::from_engine(hmac).into_inner()
+                               };
+                               $pk.mul_tweak(secp_ctx, &Scalar::from_be_bytes(hop_pk_blinding_factor).unwrap())?
+                       };
+                       let onion_packet_ss = SharedSecret::new(&blinded_hop_pk, &onion_packet_pubkey_priv);
+
+                       let rho = onion_utils::gen_rho_from_shared_secret(encrypted_data_ss.as_ref());
+                       let unblinded_pk_opt = if $blinded { None } else { Some($pk) };
+                       callback(blinded_hop_pk, onion_packet_ss, onion_packet_pubkey, rho, unblinded_pk_opt, $encrypted_payload);
+                       (encrypted_data_ss, onion_packet_ss)
+               }}
+       }
+
+       macro_rules! build_keys_in_loop {
+               ($pk: expr, $blinded: expr, $encrypted_payload: expr) => {
+                       let (encrypted_data_ss, onion_packet_ss) = build_keys!($pk, $blinded, $encrypted_payload);
+
+                       let msg_blinding_point_blinding_factor = {
+                               let mut sha = Sha256::engine();
+                               sha.input(&msg_blinding_point.serialize()[..]);
+                               sha.input(encrypted_data_ss.as_ref());
+                               Sha256::from_engine(sha).into_inner()
+                       };
+
+                       msg_blinding_point_priv = msg_blinding_point_priv.mul_tweak(&Scalar::from_be_bytes(msg_blinding_point_blinding_factor).unwrap())?;
+                       msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv);
+
+                       let onion_packet_pubkey_blinding_factor = {
+                               let mut sha = Sha256::engine();
+                               sha.input(&onion_packet_pubkey.serialize()[..]);
+                               sha.input(onion_packet_ss.as_ref());
+                               Sha256::from_engine(sha).into_inner()
+                       };
+                       onion_packet_pubkey_priv = onion_packet_pubkey_priv.mul_tweak(&Scalar::from_be_bytes(onion_packet_pubkey_blinding_factor).unwrap())?;
+                       onion_packet_pubkey = PublicKey::from_secret_key(secp_ctx, &onion_packet_pubkey_priv);
+               };
+       }
+
+       for pk in unblinded_path {
+               build_keys_in_loop!(*pk, false, None);
+       }
+       if let Some(dest) = destination {
+               match dest {
+                       Destination::Node(pk) => {
+                               build_keys!(pk, false, None);
+                       },
+                       Destination::BlindedPath(BlindedPath { blinded_hops, .. }) => {
+                               for hop in blinded_hops {
+                                       build_keys_in_loop!(hop.blinded_node_id, true, Some(hop.encrypted_payload));
+                               }
+                       },
+               }
+       }
+       Ok(())
+}
index 4bfb47de402fad766b3d0feb951ababd99415556..e7c2b0f18ec001d703eb0249964b90a8a3f1a5ed 100644 (file)
@@ -37,6 +37,7 @@ use crate::events::{Event, EventHandler};
 use crate::util::atomic_counter::AtomicCounter;
 use crate::util::logger::Logger;
 use crate::util::errors::APIError;
+use crate::util::wakers::{Future, Notifier};
 use crate::ln::channelmanager::ChannelDetails;
 
 use crate::prelude::*;
@@ -216,8 +217,15 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> Deref for LockedChannelMonitor<
 /// or used independently to monitor channels remotely. See the [module-level documentation] for
 /// details.
 ///
+/// Note that `ChainMonitor` should regularly trigger rebroadcasts/fee bumps of pending claims from
+/// a force-closed channel. This is crucial in preventing certain classes of pinning attacks,
+/// detecting substantial mempool feerate changes between blocks, and ensuring reliability if
+/// broadcasting fails. We recommend invoking this every 30 seconds, or lower if running in an
+/// environment with spotty connections, like on mobile.
+///
 /// [`ChannelManager`]: crate::ln::channelmanager::ChannelManager
 /// [module-level documentation]: crate::chain::chainmonitor
+/// [`rebroadcast_pending_claims`]: Self::rebroadcast_pending_claims
 pub struct ChainMonitor<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref>
        where C::Target: chain::Filter,
         T::Target: BroadcasterInterface,
@@ -240,6 +248,8 @@ pub struct ChainMonitor<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T:
        pending_monitor_events: Mutex<Vec<(OutPoint, Vec<MonitorEvent>, Option<PublicKey>)>>,
        /// The best block height seen, used as a proxy for the passage of time.
        highest_chain_height: AtomicUsize,
+
+       event_notifier: Notifier,
 }
 
 impl<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref> ChainMonitor<ChannelSigner, C, T, F, L, P>
@@ -300,6 +310,7 @@ where C::Target: chain::Filter,
                                        ChannelMonitorUpdateStatus::PermanentFailure => {
                                                monitor_state.channel_perm_failed.store(true, Ordering::Release);
                                                self.pending_monitor_events.lock().unwrap().push((*funding_outpoint, vec![MonitorEvent::UpdateFailed(*funding_outpoint)], monitor.get_counterparty_node_id()));
+                                               self.event_notifier.notify();
                                        },
                                        ChannelMonitorUpdateStatus::InProgress => {
                                                log_debug!(self.logger, "Channel Monitor sync for channel {} in progress, holding events until completion!", log_funding_info!(monitor));
@@ -345,6 +356,7 @@ where C::Target: chain::Filter,
                        persister,
                        pending_monitor_events: Mutex::new(Vec::new()),
                        highest_chain_height: AtomicUsize::new(0),
+                       event_notifier: Notifier::new(),
                }
        }
 
@@ -472,6 +484,7 @@ where C::Target: chain::Filter,
                                }
                        },
                }
+               self.event_notifier.notify();
                Ok(())
        }
 
@@ -486,6 +499,7 @@ where C::Target: chain::Filter,
                        funding_txo,
                        monitor_update_id,
                }], counterparty_node_id));
+               self.event_notifier.notify();
        }
 
        #[cfg(any(test, fuzzing, feature = "_test_utils"))]
@@ -514,6 +528,32 @@ where C::Target: chain::Filter,
                        handler(event).await;
                }
        }
+
+       /// Gets a [`Future`] that completes when an event is available either via
+       /// [`chain::Watch::release_pending_monitor_events`] or
+       /// [`EventsProvider::process_pending_events`].
+       ///
+       /// Note that callbacks registered on the [`Future`] MUST NOT call back into this
+       /// [`ChainMonitor`] and should instead register actions to be taken later.
+       ///
+       /// [`EventsProvider::process_pending_events`]: crate::events::EventsProvider::process_pending_events
+       pub fn get_update_future(&self) -> Future {
+               self.event_notifier.get_future()
+       }
+
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub fn rebroadcast_pending_claims(&self) {
+               let monitors = self.monitors.read().unwrap();
+               for (_, monitor_holder) in &*monitors {
+                       monitor_holder.monitor.rebroadcast_pending_claims(
+                               &*self.broadcaster, &*self.fee_estimator, &*self.logger
+                       )
+               }
+       }
 }
 
 impl<ChannelSigner: WriteableEcdsaChannelSigner, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref>
@@ -793,7 +833,7 @@ mod tests {
        use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Watch};
        use crate::chain::channelmonitor::LATENCY_GRACE_PERIOD_BLOCKS;
        use crate::events::{Event, ClosureReason, MessageSendEvent, MessageSendEventsProvider};
-       use crate::ln::channelmanager::{PaymentSendFailure, PaymentId};
+       use crate::ln::channelmanager::{PaymentSendFailure, PaymentId, RecipientOnionFields};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs::ChannelMessageHandler;
        use crate::util::errors::APIError;
@@ -945,8 +985,9 @@ mod tests {
                // If the ChannelManager tries to update the channel, however, the ChainMonitor will pass
                // the update through to the ChannelMonitor which will refuse it (as the channel is closed).
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)),
-                       true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+                               RecipientOnionFields::secret_only(second_payment_secret), PaymentId(second_payment_hash.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(err.contains("ChannelMonitor storage failure")));
                check_added_monitors!(nodes[0], 2); // After the failure we generate a close-channel monitor update
                check_closed_broadcast!(nodes[0], true);
index 20a9ebf66e0980baeabaa214c2af12ccea11ecb6..f2c6ea18249480d981337fd530bbf8874265af71 100644 (file)
@@ -494,8 +494,12 @@ impl_writeable_tlv_based_enum_upgradable!(OnchainEvent,
 pub(crate) enum ChannelMonitorUpdateStep {
        LatestHolderCommitmentTXInfo {
                commitment_tx: HolderCommitmentTransaction,
+               /// Note that LDK after 0.0.115 supports this only containing dust HTLCs (implying the
+               /// `Signature` field is never filled in). At that point, non-dust HTLCs are implied by the
+               /// HTLC fields in `commitment_tx` and the sources passed via `nondust_htlc_sources`.
                htlc_outputs: Vec<(HTLCOutputInCommitment, Option<Signature>, Option<HTLCSource>)>,
                claimed_htlcs: Vec<(SentHTLCId, PaymentPreimage)>,
+               nondust_htlc_sources: Vec<HTLCSource>,
        },
        LatestCounterpartyCommitmentTXInfo {
                commitment_txid: Txid,
@@ -540,6 +544,7 @@ impl_writeable_tlv_based_enum_upgradable!(ChannelMonitorUpdateStep,
                (0, commitment_tx, required),
                (1, claimed_htlcs, vec_type),
                (2, htlc_outputs, vec_type),
+               (4, nondust_htlc_sources, optional_vec),
        },
        (1, LatestCounterpartyCommitmentTXInfo) => {
                (0, commitment_txid, required),
@@ -1181,7 +1186,7 @@ impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitor<Signer> {
                &self, holder_commitment_tx: HolderCommitmentTransaction,
                htlc_outputs: Vec<(HTLCOutputInCommitment, Option<Signature>, Option<HTLCSource>)>,
        ) -> Result<(), ()> {
-               self.inner.lock().unwrap().provide_latest_holder_commitment_tx(holder_commitment_tx, htlc_outputs, &Vec::new()).map_err(|_| ())
+               self.inner.lock().unwrap().provide_latest_holder_commitment_tx(holder_commitment_tx, htlc_outputs, &Vec::new(), Vec::new()).map_err(|_| ())
        }
 
        /// This is used to provide payment preimage(s) out-of-band during startup without updating the
@@ -1462,6 +1467,27 @@ impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitor<Signer> {
        pub fn current_best_block(&self) -> BestBlock {
                self.inner.lock().unwrap().best_block.clone()
        }
+
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub fn rebroadcast_pending_claims<B: Deref, F: Deref, L: Deref>(
+               &self, broadcaster: B, fee_estimator: F, logger: L,
+       )
+       where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
+       {
+               let fee_estimator = LowerBoundedFeeEstimator::new(fee_estimator);
+               let mut inner = self.inner.lock().unwrap();
+               let current_height = inner.best_block.height;
+               inner.onchain_tx_handler.rebroadcast_pending_claims(
+                       current_height, &broadcaster, &fee_estimator, &logger,
+               );
+       }
 }
 
 impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitorImpl<Signer> {
@@ -2150,7 +2176,53 @@ impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitorImpl<Signer> {
        /// is important that any clones of this channel monitor (including remote clones) by kept
        /// up-to-date as our holder commitment transaction is updated.
        /// Panics if set_on_holder_tx_csv has never been called.
-       fn provide_latest_holder_commitment_tx(&mut self, holder_commitment_tx: HolderCommitmentTransaction, htlc_outputs: Vec<(HTLCOutputInCommitment, Option<Signature>, Option<HTLCSource>)>, claimed_htlcs: &[(SentHTLCId, PaymentPreimage)]) -> Result<(), &'static str> {
+       fn provide_latest_holder_commitment_tx(&mut self, holder_commitment_tx: HolderCommitmentTransaction, mut htlc_outputs: Vec<(HTLCOutputInCommitment, Option<Signature>, Option<HTLCSource>)>, claimed_htlcs: &[(SentHTLCId, PaymentPreimage)], nondust_htlc_sources: Vec<HTLCSource>) -> Result<(), &'static str> {
+               if htlc_outputs.iter().any(|(_, s, _)| s.is_some()) {
+                       // If we have non-dust HTLCs in htlc_outputs, ensure they match the HTLCs in the
+                       // `holder_commitment_tx`. In the future, we'll no longer provide the redundant data
+                       // and just pass in source data via `nondust_htlc_sources`.
+                       debug_assert_eq!(htlc_outputs.iter().filter(|(_, s, _)| s.is_some()).count(), holder_commitment_tx.trust().htlcs().len());
+                       for (a, b) in htlc_outputs.iter().filter(|(_, s, _)| s.is_some()).map(|(h, _, _)| h).zip(holder_commitment_tx.trust().htlcs().iter()) {
+                               debug_assert_eq!(a, b);
+                       }
+                       debug_assert_eq!(htlc_outputs.iter().filter(|(_, s, _)| s.is_some()).count(), holder_commitment_tx.counterparty_htlc_sigs.len());
+                       for (a, b) in htlc_outputs.iter().filter_map(|(_, s, _)| s.as_ref()).zip(holder_commitment_tx.counterparty_htlc_sigs.iter()) {
+                               debug_assert_eq!(a, b);
+                       }
+                       debug_assert!(nondust_htlc_sources.is_empty());
+               } else {
+                       // If we don't have any non-dust HTLCs in htlc_outputs, assume they were all passed via
+                       // `nondust_htlc_sources`, building up the final htlc_outputs by combining
+                       // `nondust_htlc_sources` and the `holder_commitment_tx`
+                       #[cfg(debug_assertions)] {
+                               let mut prev = -1;
+                               for htlc in holder_commitment_tx.trust().htlcs().iter() {
+                                       assert!(htlc.transaction_output_index.unwrap() as i32 > prev);
+                                       prev = htlc.transaction_output_index.unwrap() as i32;
+                               }
+                       }
+                       debug_assert!(htlc_outputs.iter().all(|(htlc, _, _)| htlc.transaction_output_index.is_none()));
+                       debug_assert!(htlc_outputs.iter().all(|(_, sig_opt, _)| sig_opt.is_none()));
+                       debug_assert_eq!(holder_commitment_tx.trust().htlcs().len(), holder_commitment_tx.counterparty_htlc_sigs.len());
+
+                       let mut sources_iter = nondust_htlc_sources.into_iter();
+
+                       for (htlc, counterparty_sig) in holder_commitment_tx.trust().htlcs().iter()
+                               .zip(holder_commitment_tx.counterparty_htlc_sigs.iter())
+                       {
+                               if htlc.offered {
+                                       let source = sources_iter.next().expect("Non-dust HTLC sources didn't match commitment tx");
+                                       #[cfg(debug_assertions)] {
+                                               assert!(source.possibly_matches_output(htlc));
+                                       }
+                                       htlc_outputs.push((htlc.clone(), Some(counterparty_sig.clone()), Some(source)));
+                               } else {
+                                       htlc_outputs.push((htlc.clone(), Some(counterparty_sig.clone()), None));
+                               }
+                       }
+                       debug_assert!(sources_iter.next().is_none());
+               }
+
                let trusted_tx = holder_commitment_tx.trust();
                let txid = trusted_tx.txid();
                let tx_keys = trusted_tx.keys();
@@ -2283,10 +2355,10 @@ impl<Signer: WriteableEcdsaChannelSigner> ChannelMonitorImpl<Signer> {
                let bounded_fee_estimator = LowerBoundedFeeEstimator::new(&*fee_estimator);
                for update in updates.updates.iter() {
                        match update {
-                               ChannelMonitorUpdateStep::LatestHolderCommitmentTXInfo { commitment_tx, htlc_outputs, claimed_htlcs } => {
+                               ChannelMonitorUpdateStep::LatestHolderCommitmentTXInfo { commitment_tx, htlc_outputs, claimed_htlcs, nondust_htlc_sources } => {
                                        log_trace!(logger, "Updating ChannelMonitor with latest holder commitment transaction info");
                                        if self.lockdown_from_offchain { panic!(); }
-                                       if let Err(e) = self.provide_latest_holder_commitment_tx(commitment_tx.clone(), htlc_outputs.clone(), &claimed_htlcs) {
+                                       if let Err(e) = self.provide_latest_holder_commitment_tx(commitment_tx.clone(), htlc_outputs.clone(), &claimed_htlcs, nondust_htlc_sources.clone()) {
                                                log_error!(logger, "Providing latest holder commitment transaction failed/was refused:");
                                                log_error!(logger, "    {}", e);
                                                ret = Err(());
@@ -3674,8 +3746,9 @@ where
        }
 }
 
-impl<Signer: WriteableEcdsaChannelSigner, T: Deref, F: Deref, L: Deref> chain::Confirm for (ChannelMonitor<Signer>, T, F, L)
+impl<Signer: WriteableEcdsaChannelSigner, M, T: Deref, F: Deref, L: Deref> chain::Confirm for (M, T, F, L)
 where
+       M: Deref<Target = ChannelMonitor<Signer>>,
        T::Target: BroadcasterInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
@@ -4015,7 +4088,7 @@ mod tests {
        use crate::ln::{PaymentPreimage, PaymentHash};
        use crate::ln::chan_utils;
        use crate::ln::chan_utils::{HTLCOutputInCommitment, ChannelPublicKeys, ChannelTransactionParameters, HolderCommitmentTransaction, CounterpartyChannelTransactionParameters};
-       use crate::ln::channelmanager::{PaymentSendFailure, PaymentId};
+       use crate::ln::channelmanager::{PaymentSendFailure, PaymentId, RecipientOnionFields};
        use crate::ln::functional_test_utils::*;
        use crate::ln::script::ShutdownScript;
        use crate::util::errors::APIError;
@@ -4077,8 +4150,9 @@ mod tests {
                // If the ChannelManager tries to update the channel, however, the ChainMonitor will pass
                // the update through to the ChannelMonitor which will refuse it (as the channel is closed).
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 100_000);
-               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
-                       true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[1].node.send_payment_with_route(&route, payment_hash,
+                               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(err.contains("ChannelMonitor storage failure")));
                check_added_monitors!(nodes[1], 2); // After the failure we generate a close-channel monitor update
                check_closed_broadcast!(nodes[1], true);
@@ -4095,7 +4169,7 @@ mod tests {
                replay_update.updates.push(ChannelMonitorUpdateStep::PaymentPreimage { payment_preimage: payment_preimage_1 });
                replay_update.updates.push(ChannelMonitorUpdateStep::PaymentPreimage { payment_preimage: payment_preimage_2 });
 
-               let broadcaster = TestBroadcaster::new(Arc::clone(&nodes[1].blocks));
+               let broadcaster = TestBroadcaster::with_blocks(Arc::clone(&nodes[1].blocks));
                assert!(
                        pre_update_monitor.update_monitor(&replay_update, &&broadcaster, &chanmon_cfgs[1].fee_estimator, &nodes[1].logger)
                        .is_err());
@@ -4121,10 +4195,7 @@ mod tests {
        fn test_prune_preimages() {
                let secp_ctx = Secp256k1::new();
                let logger = Arc::new(TestLogger::new());
-               let broadcaster = Arc::new(TestBroadcaster {
-                       txn_broadcasted: Mutex::new(Vec::new()),
-                       blocks: Arc::new(Mutex::new(Vec::new()))
-               });
+               let broadcaster = Arc::new(TestBroadcaster::new(Network::Testnet));
                let fee_estimator = TestFeeEstimator { sat_per_kw: Mutex::new(253) };
 
                let dummy_key = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
@@ -4138,7 +4209,7 @@ mod tests {
                        }
                }
 
-               macro_rules! preimages_slice_to_htlc_outputs {
+               macro_rules! preimages_slice_to_htlcs {
                        ($preimages_slice: expr) => {
                                {
                                        let mut res = Vec::new();
@@ -4149,21 +4220,20 @@ mod tests {
                                                        cltv_expiry: 0,
                                                        payment_hash: preimage.1.clone(),
                                                        transaction_output_index: Some(idx as u32),
-                                               }, None));
+                                               }, ()));
                                        }
                                        res
                                }
                        }
                }
-               macro_rules! preimages_to_holder_htlcs {
+               macro_rules! preimages_slice_to_htlc_outputs {
                        ($preimages_slice: expr) => {
-                               {
-                                       let mut inp = preimages_slice_to_htlc_outputs!($preimages_slice);
-                                       let res: Vec<_> = inp.drain(..).map(|e| { (e.0, None, e.1) }).collect();
-                                       res
-                               }
+                               preimages_slice_to_htlcs!($preimages_slice).into_iter().map(|(htlc, _)| (htlc, None)).collect()
                        }
                }
+               let dummy_sig = crate::util::crypto::sign(&secp_ctx,
+                       &bitcoin::secp256k1::Message::from_slice(&[42; 32]).unwrap(),
+                       &SecretKey::from_slice(&[42; 32]).unwrap());
 
                macro_rules! test_preimages_exist {
                        ($preimages_slice: expr, $monitor: expr) => {
@@ -4183,6 +4253,7 @@ mod tests {
                        [41; 32],
                        0,
                        [0; 32],
+                       [0; 32],
                );
 
                let counterparty_pubkeys = ChannelPublicKeys {
@@ -4210,13 +4281,15 @@ mod tests {
                let shutdown_pubkey = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
                let best_block = BestBlock::from_network(Network::Testnet);
                let monitor = ChannelMonitor::new(Secp256k1::new(), keys,
-                                                 Some(ShutdownScript::new_p2wpkh_from_pubkey(shutdown_pubkey).into_inner()), 0, &Script::new(),
-                                                 (OutPoint { txid: Txid::from_slice(&[43; 32]).unwrap(), index: 0 }, Script::new()),
-                                                 &channel_parameters,
-                                                 Script::new(), 46, 0,
-                                                 HolderCommitmentTransaction::dummy(), best_block, dummy_key);
-
-               monitor.provide_latest_holder_commitment_tx(HolderCommitmentTransaction::dummy(), preimages_to_holder_htlcs!(preimages[0..10])).unwrap();
+                       Some(ShutdownScript::new_p2wpkh_from_pubkey(shutdown_pubkey).into_inner()), 0, &Script::new(),
+                       (OutPoint { txid: Txid::from_slice(&[43; 32]).unwrap(), index: 0 }, Script::new()),
+                       &channel_parameters, Script::new(), 46, 0, HolderCommitmentTransaction::dummy(&mut Vec::new()),
+                       best_block, dummy_key);
+
+               let mut htlcs = preimages_slice_to_htlcs!(preimages[0..10]);
+               let dummy_commitment_tx = HolderCommitmentTransaction::dummy(&mut htlcs);
+               monitor.provide_latest_holder_commitment_tx(dummy_commitment_tx.clone(),
+                       htlcs.into_iter().map(|(htlc, _)| (htlc, Some(dummy_sig), None)).collect()).unwrap();
                monitor.provide_latest_counterparty_commitment_tx(Txid::from_inner(Sha256::hash(b"1").into_inner()),
                        preimages_slice_to_htlc_outputs!(preimages[5..15]), 281474976710655, dummy_key, &logger);
                monitor.provide_latest_counterparty_commitment_tx(Txid::from_inner(Sha256::hash(b"2").into_inner()),
@@ -4249,7 +4322,10 @@ mod tests {
 
                // Now update holder commitment tx info, pruning only element 18 as we still care about the
                // previous commitment tx's preimages too
-               monitor.provide_latest_holder_commitment_tx(HolderCommitmentTransaction::dummy(), preimages_to_holder_htlcs!(preimages[0..5])).unwrap();
+               let mut htlcs = preimages_slice_to_htlcs!(preimages[0..5]);
+               let dummy_commitment_tx = HolderCommitmentTransaction::dummy(&mut htlcs);
+               monitor.provide_latest_holder_commitment_tx(dummy_commitment_tx.clone(),
+                       htlcs.into_iter().map(|(htlc, _)| (htlc, Some(dummy_sig), None)).collect()).unwrap();
                secret[0..32].clone_from_slice(&hex::decode("2273e227a5b7449b6e70f1fb4652864038b1cbf9cd7c043a7d6456b7fc275ad8").unwrap());
                monitor.provide_secret(281474976710653, secret.clone()).unwrap();
                assert_eq!(monitor.inner.lock().unwrap().payment_preimages.len(), 12);
@@ -4257,7 +4333,10 @@ mod tests {
                test_preimages_exist!(&preimages[18..20], monitor);
 
                // But if we do it again, we'll prune 5-10
-               monitor.provide_latest_holder_commitment_tx(HolderCommitmentTransaction::dummy(), preimages_to_holder_htlcs!(preimages[0..3])).unwrap();
+               let mut htlcs = preimages_slice_to_htlcs!(preimages[0..3]);
+               let dummy_commitment_tx = HolderCommitmentTransaction::dummy(&mut htlcs);
+               monitor.provide_latest_holder_commitment_tx(dummy_commitment_tx,
+                       htlcs.into_iter().map(|(htlc, _)| (htlc, Some(dummy_sig), None)).collect()).unwrap();
                secret[0..32].clone_from_slice(&hex::decode("27cddaa5624534cb6cb9d7da077cf2b22ab21e9b506fd4998a51d54502e99116").unwrap());
                monitor.provide_secret(281474976710652, secret.clone()).unwrap();
                assert_eq!(monitor.inner.lock().unwrap().payment_preimages.len(), 5);
index 4ad15db876bb4cad83106c89ba7ba835409b1544..b1290708b95f70a71ebf6193cceb1677aa3150bd 100644 (file)
@@ -32,8 +32,8 @@ use bitcoin::secp256k1::ecdsa::RecoverableSignature;
 use bitcoin::{PackedLockTime, secp256k1, Sequence, Witness};
 
 use crate::util::transaction_utils;
-use crate::util::crypto::{hkdf_extract_expand_twice, sign};
-use crate::util::ser::{Writeable, Writer, Readable};
+use crate::util::crypto::{hkdf_extract_expand_twice, sign, sign_with_aux_rand};
+use crate::util::ser::{Writeable, Writer, Readable, ReadableArgs};
 use crate::chain::transaction::OutPoint;
 #[cfg(anchors)]
 use crate::events::bump_transaction::HTLCDescriptor;
@@ -45,6 +45,7 @@ use crate::ln::script::ShutdownScript;
 
 use crate::prelude::*;
 use core::convert::TryInto;
+use core::ops::Deref;
 use core::sync::atomic::{AtomicUsize, Ordering};
 use crate::io::{self, Error};
 use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
@@ -553,7 +554,6 @@ pub trait SignerProvider {
        fn get_shutdown_scriptpubkey(&self) -> ShutdownScript;
 }
 
-#[derive(Clone)]
 /// A simple implementation of [`WriteableEcdsaChannelSigner`] that just keeps the private keys in memory.
 ///
 /// This implementation performs no policy checks and is insufficient by itself as
@@ -580,6 +580,30 @@ pub struct InMemorySigner {
        channel_value_satoshis: u64,
        /// Key derivation parameters.
        channel_keys_id: [u8; 32],
+       /// Seed from which all randomness produced is derived from.
+       rand_bytes_unique_start: [u8; 32],
+       /// Tracks the number of times we've produced randomness to ensure we don't return the same
+       /// bytes twice.
+       rand_bytes_index: AtomicCounter,
+}
+
+impl Clone for InMemorySigner {
+       fn clone(&self) -> Self {
+               Self {
+                       funding_key: self.funding_key.clone(),
+                       revocation_base_key: self.revocation_base_key.clone(),
+                       payment_key: self.payment_key.clone(),
+                       delayed_payment_base_key: self.delayed_payment_base_key.clone(),
+                       htlc_base_key: self.htlc_base_key.clone(),
+                       commitment_seed: self.commitment_seed.clone(),
+                       holder_channel_pubkeys: self.holder_channel_pubkeys.clone(),
+                       channel_parameters: self.channel_parameters.clone(),
+                       channel_value_satoshis: self.channel_value_satoshis,
+                       channel_keys_id: self.channel_keys_id,
+                       rand_bytes_unique_start: self.get_secure_random_bytes(),
+                       rand_bytes_index: AtomicCounter::new(),
+               }
+       }
 }
 
 impl InMemorySigner {
@@ -594,6 +618,7 @@ impl InMemorySigner {
                commitment_seed: [u8; 32],
                channel_value_satoshis: u64,
                channel_keys_id: [u8; 32],
+               rand_bytes_unique_start: [u8; 32],
        ) -> InMemorySigner {
                let holder_channel_pubkeys =
                        InMemorySigner::make_holder_keys(secp_ctx, &funding_key, &revocation_base_key,
@@ -610,6 +635,8 @@ impl InMemorySigner {
                        holder_channel_pubkeys,
                        channel_parameters: None,
                        channel_keys_id,
+                       rand_bytes_unique_start,
+                       rand_bytes_index: AtomicCounter::new(),
                }
        }
 
@@ -686,7 +713,7 @@ impl InMemorySigner {
                let remotepubkey = self.pubkeys().payment_point;
                let witness_script = bitcoin::Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: remotepubkey}, Network::Testnet).script_pubkey();
                let sighash = hash_to_message!(&sighash::SighashCache::new(spend_tx).segwit_signature_hash(input_idx, &witness_script, descriptor.output.value, EcdsaSighashType::All).unwrap()[..]);
-               let remotesig = sign(secp_ctx, &sighash, &self.payment_key);
+               let remotesig = sign_with_aux_rand(secp_ctx, &sighash, &self.payment_key, &self);
                let payment_script = bitcoin::Address::p2wpkh(&::bitcoin::PublicKey{compressed: true, inner: remotepubkey}, Network::Bitcoin).unwrap().script_pubkey();
 
                if payment_script != descriptor.output.script_pubkey { return Err(()); }
@@ -722,7 +749,7 @@ impl InMemorySigner {
                let delayed_payment_pubkey = PublicKey::from_secret_key(&secp_ctx, &delayed_payment_key);
                let witness_script = chan_utils::get_revokeable_redeemscript(&descriptor.revocation_pubkey, descriptor.to_self_delay, &delayed_payment_pubkey);
                let sighash = hash_to_message!(&sighash::SighashCache::new(spend_tx).segwit_signature_hash(input_idx, &witness_script, descriptor.output.value, EcdsaSighashType::All).unwrap()[..]);
-               let local_delayedsig = sign(secp_ctx, &sighash, &delayed_payment_key);
+               let local_delayedsig = sign_with_aux_rand(secp_ctx, &sighash, &delayed_payment_key, &self);
                let payment_script = bitcoin::Address::p2wsh(&witness_script, Network::Bitcoin).script_pubkey();
 
                if descriptor.output.script_pubkey != payment_script { return Err(()); }
@@ -736,6 +763,15 @@ impl InMemorySigner {
        }
 }
 
+impl EntropySource for InMemorySigner {
+       fn get_secure_random_bytes(&self) -> [u8; 32] {
+               let index = self.rand_bytes_index.get_increment();
+               let mut nonce = [0u8; 16];
+               nonce[..8].copy_from_slice(&index.to_be_bytes());
+               ChaCha20::get_single_block(&self.rand_bytes_unique_start, &nonce)
+       }
+}
+
 impl ChannelSigner for InMemorySigner {
        fn get_per_commitment_point(&self, idx: u64, secp_ctx: &Secp256k1<secp256k1::All>) -> PublicKey {
                let commitment_secret = SecretKey::from_slice(&chan_utils::build_commitment_secret(&self.commitment_seed, idx)).unwrap();
@@ -774,7 +810,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let channel_funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
 
                let built_tx = trusted_tx.built_transaction();
-               let commitment_sig = built_tx.sign(&self.funding_key, &channel_funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let commitment_sig = built_tx.sign_counterparty_commitment(&self.funding_key, &channel_funding_redeemscript, self.channel_value_satoshis, secp_ctx);
                let commitment_txid = built_tx.txid;
 
                let mut htlc_sigs = Vec::with_capacity(commitment_tx.htlcs().len());
@@ -799,9 +835,9 @@ impl EcdsaChannelSigner for InMemorySigner {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
-               let sig = trusted_tx.built_transaction().sign(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let sig = trusted_tx.built_transaction().sign_holder_commitment(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, &self, secp_ctx);
                let channel_parameters = self.get_channel_parameters();
-               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), secp_ctx)?;
+               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), &self, secp_ctx)?;
                Ok((sig, htlc_sigs))
        }
 
@@ -810,9 +846,9 @@ impl EcdsaChannelSigner for InMemorySigner {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
                let trusted_tx = commitment_tx.trust();
-               let sig = trusted_tx.built_transaction().sign(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, secp_ctx);
+               let sig = trusted_tx.built_transaction().sign_holder_commitment(&self.funding_key, &funding_redeemscript, self.channel_value_satoshis, &self, secp_ctx);
                let channel_parameters = self.get_channel_parameters();
-               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), secp_ctx)?;
+               let htlc_sigs = trusted_tx.get_htlc_sigs(&self.htlc_base_key, &channel_parameters.as_holder_broadcastable(), &self, secp_ctx)?;
                Ok((sig, htlc_sigs))
        }
 
@@ -826,7 +862,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               return Ok(sign(secp_ctx, &sighash, &revocation_key))
+               return Ok(sign_with_aux_rand(secp_ctx, &sighash, &revocation_key, &self))
        }
 
        fn sign_justice_revoked_htlc(&self, justice_tx: &Transaction, input: usize, amount: u64, per_commitment_key: &SecretKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -840,7 +876,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                };
                let mut sighash_parts = sighash::SighashCache::new(justice_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               return Ok(sign(secp_ctx, &sighash, &revocation_key))
+               return Ok(sign_with_aux_rand(secp_ctx, &sighash, &revocation_key, &self))
        }
 
        #[cfg(anchors)]
@@ -858,7 +894,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let our_htlc_private_key = chan_utils::derive_private_key(
                        &secp_ctx, &per_commitment_point, &self.htlc_base_key
                );
-               Ok(sign(&secp_ctx, &hash_to_message!(sighash), &our_htlc_private_key))
+               Ok(sign_with_aux_rand(&secp_ctx, &hash_to_message!(sighash), &our_htlc_private_key, &self))
        }
 
        fn sign_counterparty_htlc_transaction(&self, htlc_tx: &Transaction, input: usize, amount: u64, per_commitment_point: &PublicKey, htlc: &HTLCOutputInCommitment, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -869,7 +905,7 @@ impl EcdsaChannelSigner for InMemorySigner {
                let witness_script = chan_utils::get_htlc_redeemscript_with_explicit_keys(&htlc, self.opt_anchors(), &counterparty_htlcpubkey, &htlcpubkey, &revocation_pubkey);
                let mut sighash_parts = sighash::SighashCache::new(htlc_tx);
                let sighash = hash_to_message!(&sighash_parts.segwit_signature_hash(input, &witness_script, amount, EcdsaSighashType::All).unwrap()[..]);
-               Ok(sign(secp_ctx, &sighash, &htlc_key))
+               Ok(sign_with_aux_rand(secp_ctx, &sighash, &htlc_key, &self))
        }
 
        fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
@@ -885,14 +921,14 @@ impl EcdsaChannelSigner for InMemorySigner {
                let sighash = sighash::SighashCache::new(&*anchor_tx).segwit_signature_hash(
                        input, &witness_script, ANCHOR_OUTPUT_VALUE_SATOSHI, EcdsaSighashType::All,
                ).unwrap();
-               Ok(sign(secp_ctx, &hash_to_message!(&sighash[..]), &self.funding_key))
+               Ok(sign_with_aux_rand(secp_ctx, &hash_to_message!(&sighash[..]), &self.funding_key, &self))
        }
 
        fn sign_channel_announcement_with_funding_key(
                &self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>
        ) -> Result<Signature, ()> {
                let msghash = hash_to_message!(&Sha256dHash::hash(&msg.encode()[..])[..]);
-               Ok(sign(secp_ctx, &msghash, &self.funding_key))
+               Ok(secp_ctx.sign_ecdsa(&msghash, &self.funding_key))
        }
 }
 
@@ -922,8 +958,8 @@ impl Writeable for InMemorySigner {
        }
 }
 
-impl Readable for InMemorySigner {
-       fn read<R: io::Read>(reader: &mut R) -> Result<Self, DecodeError> {
+impl<ES: Deref> ReadableArgs<ES> for InMemorySigner where ES::Target: EntropySource {
+       fn read<R: io::Read>(reader: &mut R, entropy_source: ES) -> Result<Self, DecodeError> {
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
 
                let funding_key = Readable::read(reader)?;
@@ -953,6 +989,8 @@ impl Readable for InMemorySigner {
                        holder_channel_pubkeys,
                        channel_parameters: counterparty_channel_data,
                        channel_keys_id: keys_id,
+                       rand_bytes_unique_start: entropy_source.get_secure_random_bytes(),
+                       rand_bytes_index: AtomicCounter::new(),
                })
        }
 }
@@ -1107,6 +1145,7 @@ impl KeysManager {
                let payment_key = key_step!(b"payment key", revocation_base_key);
                let delayed_payment_base_key = key_step!(b"delayed payment base key", payment_key);
                let htlc_base_key = key_step!(b"HTLC base key", delayed_payment_base_key);
+               let prng_seed = self.get_secure_random_bytes();
 
                InMemorySigner::new(
                        &self.secp_ctx,
@@ -1118,6 +1157,7 @@ impl KeysManager {
                        commitment_seed,
                        channel_value_satoshis,
                        params.clone(),
+                       prng_seed,
                )
        }
 
@@ -1233,7 +1273,7 @@ impl KeysManager {
                                        if payment_script != output.script_pubkey { return Err(()); };
 
                                        let sighash = hash_to_message!(&sighash::SighashCache::new(&spend_tx).segwit_signature_hash(input_idx, &witness_script, output.value, EcdsaSighashType::All).unwrap()[..]);
-                                       let sig = sign(secp_ctx, &sighash, &secret.private_key);
+                                       let sig = sign_with_aux_rand(secp_ctx, &sighash, &secret.private_key, &self);
                                        let mut sig_ser = sig.serialize_der().to_vec();
                                        sig_ser.push(EcdsaSighashType::All as u8);
                                        spend_tx.input[input_idx].witness.push(sig_ser);
@@ -1295,7 +1335,7 @@ impl NodeSigner for KeysManager {
 
        fn sign_gossip_message(&self, msg: UnsignedGossipMessage) -> Result<Signature, ()> {
                let msg_hash = hash_to_message!(&Sha256dHash::hash(&msg.encode()[..])[..]);
-               Ok(sign(&self.secp_ctx, &msg_hash, &self.node_secret))
+               Ok(self.secp_ctx.sign_ecdsa(&msg_hash, &self.node_secret))
        }
 }
 
@@ -1323,7 +1363,7 @@ impl SignerProvider for KeysManager {
        }
 
        fn read_chan_signer(&self, reader: &[u8]) -> Result<Self::Signer, DecodeError> {
-               InMemorySigner::read(&mut io::Cursor::new(reader))
+               InMemorySigner::read(&mut io::Cursor::new(reader), self)
        }
 
        fn get_destination_script(&self) -> Script {
index f690d3664273b94e8ee9fe8d6d4c9f16d44ece0e..2d5c85cd477e56660f5a32cfd2a7b95318ba1c57 100644 (file)
@@ -481,6 +481,59 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                events.into_iter().map(|(_, event)| event).collect()
        }
 
+       /// Triggers rebroadcasts/fee-bumps of pending claims from a force-closed channel. This is
+       /// crucial in preventing certain classes of pinning attacks, detecting substantial mempool
+       /// feerate changes between blocks, and ensuring reliability if broadcasting fails. We recommend
+       /// invoking this every 30 seconds, or lower if running in an environment with spotty
+       /// connections, like on mobile.
+       pub(crate) fn rebroadcast_pending_claims<B: Deref, F: Deref, L: Deref>(
+               &mut self, current_height: u32, broadcaster: &B, fee_estimator: &LowerBoundedFeeEstimator<F>,
+               logger: &L,
+       )
+       where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
+       {
+               let mut bump_requests = Vec::with_capacity(self.pending_claim_requests.len());
+               for (package_id, request) in self.pending_claim_requests.iter() {
+                       let inputs = request.outpoints();
+                       log_info!(logger, "Triggering rebroadcast/fee-bump for request with inputs {:?}", inputs);
+                       bump_requests.push((*package_id, request.clone()));
+               }
+               for (package_id, request) in bump_requests {
+                       self.generate_claim(current_height, &request, false /* force_feerate_bump */, fee_estimator, logger)
+                               .map(|(_, new_feerate, claim)| {
+                                       let mut bumped_feerate = false;
+                                       if let Some(mut_request) = self.pending_claim_requests.get_mut(&package_id) {
+                                               bumped_feerate = request.previous_feerate() > new_feerate;
+                                               mut_request.set_feerate(new_feerate);
+                                       }
+                                       match claim {
+                                               OnchainClaim::Tx(tx) => {
+                                                       let log_start = if bumped_feerate { "Broadcasting RBF-bumped" } else { "Rebroadcasting" };
+                                                       log_info!(logger, "{} onchain {}", log_start, log_tx!(tx));
+                                                       broadcaster.broadcast_transaction(&tx);
+                                               },
+                                               #[cfg(anchors)]
+                                               OnchainClaim::Event(event) => {
+                                                       let log_start = if bumped_feerate { "Yielding fee-bumped" } else { "Replaying" };
+                                                       log_info!(logger, "{} onchain event to spend inputs {:?}", log_start,
+                                                               request.outpoints());
+                                                       #[cfg(debug_assertions)] {
+                                                               debug_assert!(request.requires_external_funding());
+                                                               let num_existing = self.pending_claim_events.iter()
+                                                                       .filter(|entry| entry.0 == package_id).count();
+                                                               assert!(num_existing == 0 || num_existing == 1);
+                                                       }
+                                                       self.pending_claim_events.retain(|event| event.0 != package_id);
+                                                       self.pending_claim_events.push((package_id, event));
+                                               }
+                                       }
+                               });
+               }
+       }
+
        /// Lightning security model (i.e being able to redeem/timeout HTLC or penalize counterparty
        /// onchain) lays on the assumption of claim transactions getting confirmed before timelock
        /// expiration (CSV or CLTV following cases). In case of high-fee spikes, claim tx may get stuck
@@ -489,9 +542,13 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
        ///
        /// Panics if there are signing errors, because signing operations in reaction to on-chain
        /// events are not expected to fail, and if they do, we may lose funds.
-       fn generate_claim<F: Deref, L: Deref>(&mut self, cur_height: u32, cached_request: &PackageTemplate, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(Option<u32>, u64, OnchainClaim)>
-               where F::Target: FeeEstimator,
-                                       L::Target: Logger,
+       fn generate_claim<F: Deref, L: Deref>(
+               &mut self, cur_height: u32, cached_request: &PackageTemplate, force_feerate_bump: bool,
+               fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+       ) -> Option<(u32, u64, OnchainClaim)>
+       where
+               F::Target: FeeEstimator,
+               L::Target: Logger,
        {
                let request_outpoints = cached_request.outpoints();
                if request_outpoints.is_empty() {
@@ -533,13 +590,14 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                // Compute new height timer to decide when we need to regenerate a new bumped version of the claim tx (if we
                // didn't receive confirmation of it before, or not enough reorg-safe depth on top of it).
-               let new_timer = Some(cached_request.get_height_timer(cur_height));
+               let new_timer = cached_request.get_height_timer(cur_height);
                if cached_request.is_malleable() {
                        #[cfg(anchors)]
                        { // Attributes are not allowed on if expressions on our current MSRV of 1.41.
                                if cached_request.requires_external_funding() {
-                                       let target_feerate_sat_per_1000_weight = cached_request
-                                               .compute_package_feerate(fee_estimator, ConfirmationTarget::HighPriority);
+                                       let target_feerate_sat_per_1000_weight = cached_request.compute_package_feerate(
+                                               fee_estimator, ConfirmationTarget::HighPriority, force_feerate_bump
+                                       );
                                        if let Some(htlcs) = cached_request.construct_malleable_package_with_external_funding(self) {
                                                return Some((
                                                        new_timer,
@@ -558,14 +616,15 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                        let predicted_weight = cached_request.package_weight(&self.destination_script);
                        if let Some((output_value, new_feerate)) = cached_request.compute_package_output(
-                               predicted_weight, self.destination_script.dust_value().to_sat(), fee_estimator, logger,
+                               predicted_weight, self.destination_script.dust_value().to_sat(),
+                               force_feerate_bump, fee_estimator, logger,
                        ) {
                                assert!(new_feerate != 0);
 
                                let transaction = cached_request.finalize_malleable_package(
                                        cur_height, self, output_value, self.destination_script.clone(), logger
                                ).unwrap();
-                               log_trace!(logger, "...with timer {} and feerate {}", new_timer.unwrap(), new_feerate);
+                               log_trace!(logger, "...with timer {} and feerate {}", new_timer, new_feerate);
                                assert!(predicted_weight >= transaction.weight());
                                return Some((new_timer, new_feerate, OnchainClaim::Tx(transaction)));
                        }
@@ -583,7 +642,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                None => return None,
                        };
                        if !cached_request.requires_external_funding() {
-                               return Some((None, 0, OnchainClaim::Tx(tx)));
+                               return Some((new_timer, 0, OnchainClaim::Tx(tx)));
                        }
                        #[cfg(anchors)]
                        return inputs.find_map(|input| match input {
@@ -601,7 +660,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                                        // counterparty's latest commitment don't have any HTLCs present.
                                                        let conf_target = ConfirmationTarget::HighPriority;
                                                        let package_target_feerate_sat_per_1000_weight = cached_request
-                                                               .compute_package_feerate(fee_estimator, conf_target);
+                                                               .compute_package_feerate(fee_estimator, conf_target, force_feerate_bump);
                                                        Some((
                                                                new_timer,
                                                                package_target_feerate_sat_per_1000_weight as u64,
@@ -616,7 +675,7 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                                                // attempt to broadcast the transaction with its current fee rate and hope
                                                // it confirms. This is essentially the same behavior as a commitment
                                                // transaction without anchor outputs.
-                                               None => Some((None, 0, OnchainClaim::Tx(tx.clone()))),
+                                               None => Some((new_timer, 0, OnchainClaim::Tx(tx.clone()))),
                                        }
                                },
                                _ => {
@@ -689,8 +748,8 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                        preprocessed_requests.push(req);
                }
 
-               // Claim everything up to and including cur_height + 1
-               let remaining_locked_packages = self.locktimed_packages.split_off(&(cur_height + 2));
+               // Claim everything up to and including `cur_height`
+               let remaining_locked_packages = self.locktimed_packages.split_off(&(cur_height + 1));
                for (pop_height, mut entry) in self.locktimed_packages.iter_mut() {
                        log_trace!(logger, "Restoring delayed claim of package(s) at their timelock at {}.", pop_height);
                        preprocessed_requests.append(&mut entry);
@@ -700,7 +759,9 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                // Generate claim transactions and track them to bump if necessary at
                // height timer expiration (i.e in how many blocks we're going to take action).
                for mut req in preprocessed_requests {
-                       if let Some((new_timer, new_feerate, claim)) = self.generate_claim(cur_height, &req, &*fee_estimator, &*logger) {
+                       if let Some((new_timer, new_feerate, claim)) = self.generate_claim(
+                               cur_height, &req, true /* force_feerate_bump */, &*fee_estimator, &*logger,
+                       ) {
                                req.set_timer(new_timer);
                                req.set_feerate(new_feerate);
                                let package_id = match claim {
@@ -885,17 +946,17 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
 
                // Check if any pending claim request must be rescheduled
                for (package_id, request) in self.pending_claim_requests.iter() {
-                       if let Some(h) = request.timer() {
-                               if cur_height >= h {
-                                       bump_candidates.insert(*package_id, request.clone());
-                               }
+                       if cur_height >= request.timer() {
+                               bump_candidates.insert(*package_id, request.clone());
                        }
                }
 
                // Build, bump and rebroadcast tx accordingly
                log_trace!(logger, "Bumping {} candidates", bump_candidates.len());
                for (package_id, request) in bump_candidates.iter() {
-                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(cur_height, &request, &*fee_estimator, &*logger) {
+                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(
+                               cur_height, &request, true /* force_feerate_bump */, &*fee_estimator, &*logger,
+                       ) {
                                match bump_claim {
                                        OnchainClaim::Tx(bump_tx) => {
                                                log_info!(logger, "Broadcasting RBF-bumped onchain {}", log_tx!(bump_tx));
@@ -975,7 +1036,11 @@ impl<ChannelSigner: WriteableEcdsaChannelSigner> OnchainTxHandler<ChannelSigner>
                        }
                }
                for ((_package_id, _), ref mut request) in bump_candidates.iter_mut() {
-                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(height, &request, fee_estimator, &&*logger) {
+                       // `height` is the height being disconnected, so our `current_height` is 1 lower.
+                       let current_height = height - 1;
+                       if let Some((new_timer, new_feerate, bump_claim)) = self.generate_claim(
+                               current_height, &request, true /* force_feerate_bump */, fee_estimator, &&*logger
+                       ) {
                                request.set_timer(new_timer);
                                request.set_feerate(new_feerate);
                                match bump_claim {
index e8886e5a2aa49958fcac012529a3ae4160348230..7ea61dc244976c2257087b91ade479d5aa758832 100644 (file)
@@ -460,13 +460,13 @@ impl PackageSolvingData {
                }
        }
        fn absolute_tx_timelock(&self, current_height: u32) -> u32 {
-               // We use `current_height + 1` as our default locktime to discourage fee sniping and because
+               // We use `current_height` as our default locktime to discourage fee sniping and because
                // transactions with it always propagate.
                let absolute_timelock = match self {
-                       PackageSolvingData::RevokedOutput(_) => current_height + 1,
-                       PackageSolvingData::RevokedHTLCOutput(_) => current_height + 1,
-                       PackageSolvingData::CounterpartyOfferedHTLCOutput(_) => current_height + 1,
-                       PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => cmp::max(outp.htlc.cltv_expiry, current_height + 1),
+                       PackageSolvingData::RevokedOutput(_) => current_height,
+                       PackageSolvingData::RevokedHTLCOutput(_) => current_height,
+                       PackageSolvingData::CounterpartyOfferedHTLCOutput(_) => current_height,
+                       PackageSolvingData::CounterpartyReceivedHTLCOutput(ref outp) => cmp::max(outp.htlc.cltv_expiry, current_height),
                        // HTLC timeout/success transactions rely on a fixed timelock due to the counterparty's
                        // signature.
                        PackageSolvingData::HolderHTLCOutput(ref outp) => {
@@ -475,7 +475,7 @@ impl PackageSolvingData {
                                }
                                outp.cltv_expiry
                        },
-                       PackageSolvingData::HolderFundingOutput(_) => current_height + 1,
+                       PackageSolvingData::HolderFundingOutput(_) => current_height,
                };
                absolute_timelock
        }
@@ -538,7 +538,7 @@ pub struct PackageTemplate {
        feerate_previous: u64,
        // Cache of next height at which fee-bumping and rebroadcast will be attempted. In
        // the future, we might abstract it to an observed mempool fluctuation.
-       height_timer: Option<u32>,
+       height_timer: u32,
        // Confirmation height of the claimed outputs set transaction. In case of reorg reaching
        // it, we wipe out and forget the package.
        height_original: u32,
@@ -554,16 +554,16 @@ impl PackageTemplate {
        pub(crate) fn aggregable(&self) -> bool {
                self.aggregable
        }
+       pub(crate) fn previous_feerate(&self) -> u64 {
+               self.feerate_previous
+       }
        pub(crate) fn set_feerate(&mut self, new_feerate: u64) {
                self.feerate_previous = new_feerate;
        }
-       pub(crate) fn timer(&self) -> Option<u32> {
-               if let Some(ref timer) = self.height_timer {
-                       return Some(*timer);
-               }
-               None
+       pub(crate) fn timer(&self) -> u32 {
+               self.height_timer
        }
-       pub(crate) fn set_timer(&mut self, new_timer: Option<u32>) {
+       pub(crate) fn set_timer(&mut self, new_timer: u32) {
                self.height_timer = new_timer;
        }
        pub(crate) fn outpoints(&self) -> Vec<&BitcoinOutPoint> {
@@ -765,16 +765,23 @@ impl PackageTemplate {
        /// Returns value in satoshis to be included as package outgoing output amount and feerate
        /// which was used to generate the value. Will not return less than `dust_limit_sats` for the
        /// value.
-       pub(crate) fn compute_package_output<F: Deref, L: Deref>(&self, predicted_weight: usize, dust_limit_sats: u64, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(u64, u64)>
-               where F::Target: FeeEstimator,
-                     L::Target: Logger,
+       pub(crate) fn compute_package_output<F: Deref, L: Deref>(
+               &self, predicted_weight: usize, dust_limit_sats: u64, force_feerate_bump: bool,
+               fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+       ) -> Option<(u64, u64)>
+       where
+               F::Target: FeeEstimator,
+               L::Target: Logger,
        {
                debug_assert!(self.malleability == PackageMalleability::Malleable, "The package output is fixed for non-malleable packages");
                let input_amounts = self.package_amount();
                assert!(dust_limit_sats as i64 > 0, "Output script must be broadcastable/have a 'real' dust limit.");
                // If old feerate is 0, first iteration of this claim, use normal fee calculation
                if self.feerate_previous != 0 {
-                       if let Some((new_fee, feerate)) = feerate_bump(predicted_weight, input_amounts, self.feerate_previous, fee_estimator, logger) {
+                       if let Some((new_fee, feerate)) = feerate_bump(
+                               predicted_weight, input_amounts, self.feerate_previous, force_feerate_bump,
+                               fee_estimator, logger,
+                       ) {
                                return Some((cmp::max(input_amounts as i64 - new_fee as i64, dust_limit_sats as i64) as u64, feerate));
                        }
                } else {
@@ -787,16 +794,19 @@ impl PackageTemplate {
 
        #[cfg(anchors)]
        /// Computes a feerate based on the given confirmation target. If a previous feerate was used,
-       /// and the new feerate is below it, we'll use a 25% increase of the previous feerate instead of
-       /// the new one.
+       /// the new feerate is below it, and `force_feerate_bump` is set, we'll use a 25% increase of
+       /// the previous feerate instead of the new feerate.
        pub(crate) fn compute_package_feerate<F: Deref>(
                &self, fee_estimator: &LowerBoundedFeeEstimator<F>, conf_target: ConfirmationTarget,
+               force_feerate_bump: bool,
        ) -> u32 where F::Target: FeeEstimator {
                let feerate_estimate = fee_estimator.bounded_sat_per_1000_weight(conf_target);
                if self.feerate_previous != 0 {
                        // If old feerate inferior to actual one given back by Fee Estimator, use it to compute new fee...
                        if feerate_estimate as u64 > self.feerate_previous {
                                feerate_estimate
+                       } else if !force_feerate_bump {
+                               self.feerate_previous.try_into().unwrap_or(u32::max_value())
                        } else {
                                // ...else just increase the previous feerate by 25% (because that's a nice number)
                                (self.feerate_previous + (self.feerate_previous / 4)).try_into().unwrap_or(u32::max_value())
@@ -837,7 +847,7 @@ impl PackageTemplate {
                        soonest_conf_deadline,
                        aggregable,
                        feerate_previous: 0,
-                       height_timer: None,
+                       height_timer: height_original,
                        height_original,
                }
        }
@@ -854,7 +864,7 @@ impl Writeable for PackageTemplate {
                        (0, self.soonest_conf_deadline, required),
                        (2, self.feerate_previous, required),
                        (4, self.height_original, required),
-                       (6, self.height_timer, option)
+                       (6, self.height_timer, required)
                });
                Ok(())
        }
@@ -893,13 +903,16 @@ impl Readable for PackageTemplate {
                        (4, height_original, required),
                        (6, height_timer, option),
                });
+               if height_timer.is_none() {
+                       height_timer = Some(height_original);
+               }
                Ok(PackageTemplate {
                        inputs,
                        malleability,
                        soonest_conf_deadline,
                        aggregable,
                        feerate_previous,
-                       height_timer,
+                       height_timer: height_timer.unwrap(),
                        height_original,
                })
        }
@@ -945,32 +958,47 @@ fn compute_fee_from_spent_amounts<F: Deref, L: Deref>(input_amounts: u64, predic
 
 /// Attempt to propose a bumping fee for a transaction from its spent output's values and predicted
 /// weight. If feerates proposed by the fee-estimator have been increasing since last fee-bumping
-/// attempt, use them. Otherwise, blindly bump the feerate by 25% of the previous feerate. We also
-/// verify that those bumping heuristics respect BIP125 rules 3) and 4) and if required adjust
-/// the new fee to meet the RBF policy requirement.
-fn feerate_bump<F: Deref, L: Deref>(predicted_weight: usize, input_amounts: u64, previous_feerate: u64, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L) -> Option<(u64, u64)>
-       where F::Target: FeeEstimator,
-             L::Target: Logger,
+/// attempt, use them. If `force_feerate_bump` is set, we bump the feerate by 25% of the previous
+/// feerate, or just use the previous feerate otherwise. If a feerate bump did happen, we also
+/// verify that those bumping heuristics respect BIP125 rules 3) and 4) and if required adjust the
+/// new fee to meet the RBF policy requirement.
+fn feerate_bump<F: Deref, L: Deref>(
+       predicted_weight: usize, input_amounts: u64, previous_feerate: u64, force_feerate_bump: bool,
+       fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L,
+) -> Option<(u64, u64)>
+where
+       F::Target: FeeEstimator,
+       L::Target: Logger,
 {
        // If old feerate inferior to actual one given back by Fee Estimator, use it to compute new fee...
-       let new_fee = if let Some((new_fee, _)) = compute_fee_from_spent_amounts(input_amounts, predicted_weight, fee_estimator, logger) {
-               let updated_feerate = new_fee / (predicted_weight as u64 * 1000);
-               if updated_feerate > previous_feerate {
-                       new_fee
+       let (new_fee, new_feerate) = if let Some((new_fee, new_feerate)) = compute_fee_from_spent_amounts(input_amounts, predicted_weight, fee_estimator, logger) {
+               if new_feerate > previous_feerate {
+                       (new_fee, new_feerate)
+               } else if !force_feerate_bump {
+                       let previous_fee = previous_feerate * (predicted_weight as u64) / 1000;
+                       (previous_fee, previous_feerate)
                } else {
                        // ...else just increase the previous feerate by 25% (because that's a nice number)
-                       let new_fee = previous_feerate * (predicted_weight as u64) / 750;
-                       if input_amounts <= new_fee {
+                       let bumped_feerate = previous_feerate + (previous_feerate / 4);
+                       let bumped_fee = bumped_feerate * (predicted_weight as u64) / 1000;
+                       if input_amounts <= bumped_fee {
                                log_warn!(logger, "Can't 25% bump new claiming tx, amount {} is too small", input_amounts);
                                return None;
                        }
-                       new_fee
+                       (bumped_fee, bumped_feerate)
                }
        } else {
                log_warn!(logger, "Can't new-estimation bump new claiming tx, amount {} is too small", input_amounts);
                return None;
        };
 
+       // Our feerates should never decrease. If it hasn't changed though, we just need to
+       // rebroadcast/re-sign the previous claim.
+       debug_assert!(new_feerate >= previous_feerate);
+       if new_feerate == previous_feerate {
+               return Some((new_fee, new_feerate));
+       }
+
        let previous_fee = previous_feerate * (predicted_weight as u64) / 1000;
        let min_relay_fee = MIN_RELAY_FEE_SAT_PER_1000_WEIGHT * (predicted_weight as u64) / 1000;
        // BIP 125 Opt-in Full Replace-by-Fee Signaling
@@ -1177,12 +1205,9 @@ mod tests {
                let revk_outp = dumb_revk_output!(secp_ctx);
 
                let mut package = PackageTemplate::build_package(txid, 0, revk_outp, 1000, true, 100);
-               let timer_none = package.timer();
-               assert!(timer_none.is_none());
-               package.set_timer(Some(100));
-               if let Some(timer_some) = package.timer() {
-                       assert_eq!(timer_some, 100);
-               } else { panic!() }
+               assert_eq!(package.timer(), 100);
+               package.set_timer(101);
+               assert_eq!(package.timer(), 101);
        }
 
        #[test]
index a96d36b645312e6efe854a8ddb4f272db3b94532..110d56cfe4aa327318c1e1dc4cde505b7e8d3df6 100644 (file)
@@ -21,7 +21,7 @@ pub mod bump_transaction;
 pub use bump_transaction::BumpTransactionEvent;
 
 use crate::chain::keysinterface::SpendableOutputDescriptor;
-use crate::ln::channelmanager::{InterceptId, PaymentId};
+use crate::ln::channelmanager::{InterceptId, PaymentId, RecipientOnionFields};
 use crate::ln::channel::FUNDING_CONF_DEADLINE_BLOCKS;
 use crate::ln::features::ChannelTypeFeatures;
 use crate::ln::msgs;
@@ -30,9 +30,11 @@ use crate::routing::gossip::NetworkUpdate;
 use crate::util::errors::APIError;
 use crate::util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, RequiredWrapper, UpgradableRequired, WithoutLength};
 use crate::util::string::UntrustedString;
-use crate::routing::router::{RouteHop, RouteParameters};
+use crate::routing::router::{BlindedTail, Path, RouteHop, RouteParameters};
 
-use bitcoin::{PackedLockTime, Transaction};
+use bitcoin::{PackedLockTime, Transaction, OutPoint};
+#[cfg(anchors)]
+use bitcoin::{Txid, TxIn, TxOut, Witness};
 use bitcoin::blockdata::script::Script;
 use bitcoin::hashes::Hash;
 use bitcoin::hashes::sha256::Hash as Sha256;
@@ -230,8 +232,11 @@ pub enum HTLCDestination {
        ///
        /// Some of the reasons may include:
        /// * HTLC Timeouts
-       /// * Expected MPP amount has already been reached
-       /// * Claimable amount does not match expected amount
+       /// * Excess HTLCs for a payment that we have already fully received, over-paying for the
+       ///   payment,
+       /// * The counterparty node modified the HTLC in transit,
+       /// * A probing attack where an intermediary node is trying to detect if we are the ultimate
+       ///   recipient for a payment.
        FailedPayment {
                /// The payment hash of the payment we attempted to process.
                payment_hash: PaymentHash
@@ -270,6 +275,43 @@ impl_writeable_tlv_based_enum!(InterceptNextHop,
        };
 );
 
+/// The reason the payment failed. Used in [`Event::PaymentFailed`].
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum PaymentFailureReason {
+       /// The intended recipient rejected our payment.
+       RecipientRejected,
+       /// The user chose to abandon this payment by calling [`ChannelManager::abandon_payment`].
+       ///
+       /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
+       UserAbandoned,
+       /// We exhausted all of our retry attempts while trying to send the payment, or we
+       /// exhausted the [`Retry::Timeout`] if the user set one. If at any point a retry
+       /// attempt failed while being forwarded along the path, an [`Event::PaymentPathFailed`] will
+       /// have come before this.
+       ///
+       /// [`Retry::Timeout`]: crate::ln::channelmanager::Retry::Timeout
+       RetriesExhausted,
+       /// The payment expired while retrying, based on the provided
+       /// [`PaymentParameters::expiry_time`].
+       ///
+       /// [`PaymentParameters::expiry_time`]: crate::routing::router::PaymentParameters::expiry_time
+       PaymentExpired,
+       /// We failed to find a route while retrying the payment.
+       RouteNotFound,
+       /// This error should generally never happen. This likely means that there is a problem with
+       /// your router.
+       UnexpectedError,
+}
+
+impl_writeable_tlv_based_enum!(PaymentFailureReason,
+       (0, RecipientRejected) => {},
+       (2, UserAbandoned) => {},
+       (4, RetriesExhausted) => {},
+       (6, PaymentExpired) => {},
+       (8, RouteNotFound) => {},
+       (10, UnexpectedError) => {}, ;
+);
+
 /// An Event which you should probably take some action in response to.
 ///
 /// Note that while Writeable and Readable are implemented for Event, you probably shouldn't use
@@ -319,7 +361,9 @@ pub enum Event {
        ///
        /// # Note
        /// LDK will not stop an inbound payment from being paid multiple times, so multiple
-       /// `PaymentClaimable` events may be generated for the same payment.
+       /// `PaymentClaimable` events may be generated for the same payment. In such a case it is
+       /// polite (and required in the lightning specification) to fail the payment the second time
+       /// and give the sender their money back rather than accepting double payment.
        ///
        /// # Note
        /// This event used to be called `PaymentReceived` in LDK versions 0.0.112 and earlier.
@@ -338,6 +382,11 @@ pub enum Event {
                /// The hash for which the preimage should be handed to the ChannelManager. Note that LDK will
                /// not stop you from registering duplicate payment hashes for inbound payments.
                payment_hash: PaymentHash,
+               /// The fields in the onion which were received with each HTLC. Only fields which were
+               /// identical in each HTLC involved in the payment will be included here.
+               ///
+               /// Payments received on LDK versions prior to 0.0.115 will have this field unset.
+               onion_fields: Option<RecipientOnionFields>,
                /// The value, in thousandths of a satoshi, that this payment is for.
                amount_msat: u64,
                /// Information for claiming this received payment, based on whether the purpose of the
@@ -347,6 +396,14 @@ pub enum Event {
                via_channel_id: Option<[u8; 32]>,
                /// The `user_channel_id` indicating over which channel we received the payment.
                via_user_channel_id: Option<u128>,
+               /// The block height at which this payment will be failed back and will no longer be
+               /// eligible for claiming.
+               ///
+               /// Prior to this height, a call to [`ChannelManager::claim_funds`] is guaranteed to
+               /// succeed, however you should wait for [`Event::PaymentClaimed`] to be sure.
+               ///
+               /// [`ChannelManager::claim_funds`]: crate::ln::channelmanager::ChannelManager::claim_funds
+               claim_deadline: Option<u32>,
        },
        /// Indicates a payment has been claimed and we've received money!
        ///
@@ -385,7 +442,7 @@ pub enum Event {
        /// Note for MPP payments: in rare cases, this event may be preceded by a `PaymentPathFailed`
        /// event. In this situation, you SHOULD treat this payment as having succeeded.
        PaymentSent {
-               /// The id returned by [`ChannelManager::send_payment`].
+               /// The `payment_id` passed to [`ChannelManager::send_payment`].
                ///
                /// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
                payment_id: Option<PaymentId>,
@@ -418,23 +475,24 @@ pub enum Event {
        /// [`Retry`]: crate::ln::channelmanager::Retry
        /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
        PaymentFailed {
-               /// The id returned by [`ChannelManager::send_payment`] and used with
-               /// [`ChannelManager::abandon_payment`].
+               /// The `payment_id` passed to [`ChannelManager::send_payment`].
                ///
                /// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
-               /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
                payment_id: PaymentId,
                /// The hash that was given to [`ChannelManager::send_payment`].
                ///
                /// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
                payment_hash: PaymentHash,
+               /// The reason the payment failed. This is only `None` for events generated or serialized
+               /// by versions prior to 0.0.115.
+               reason: Option<PaymentFailureReason>,
        },
        /// Indicates that a path for an outbound payment was successful.
        ///
        /// Always generated after [`Event::PaymentSent`] and thus useful for scoring channels. See
        /// [`Event::PaymentSent`] for obtaining the payment preimage.
        PaymentPathSuccessful {
-               /// The id returned by [`ChannelManager::send_payment`].
+               /// The `payment_id` passed to [`ChannelManager::send_payment`].
                ///
                /// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
                payment_id: PaymentId,
@@ -445,7 +503,7 @@ pub enum Event {
                /// The payment path that was successful.
                ///
                /// May contain a closed channel if the HTLC sent along the path was fulfilled on chain.
-               path: Vec<RouteHop>,
+               path: Path,
        },
        /// Indicates an outbound HTLC we sent failed, likely due to an intermediary node being unable to
        /// handle the HTLC.
@@ -458,8 +516,7 @@ pub enum Event {
        ///
        /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
        PaymentPathFailed {
-               /// The id returned by [`ChannelManager::send_payment`] and used with
-               /// [`ChannelManager::abandon_payment`].
+               /// The `payment_id` passed to [`ChannelManager::send_payment`].
                ///
                /// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
                /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
@@ -478,7 +535,7 @@ pub enum Event {
                /// [`NetworkGraph`]: crate::routing::gossip::NetworkGraph
                failure: PathFailure,
                /// The payment path that failed.
-               path: Vec<RouteHop>,
+               path: Path,
                /// The channel responsible for the failed payment path.
                ///
                /// Note that for route hints or for the first hop in a path this may be an SCID alias and
@@ -504,7 +561,7 @@ pub enum Event {
                /// [`ChannelManager::send_probe`]: crate::ln::channelmanager::ChannelManager::send_probe
                payment_hash: PaymentHash,
                /// The payment path that was successful.
-               path: Vec<RouteHop>,
+               path: Path,
        },
        /// Indicates that a probe payment we sent failed at an intermediary node on the path.
        ProbeFailed {
@@ -517,7 +574,7 @@ pub enum Event {
                /// [`ChannelManager::send_probe`]: crate::ln::channelmanager::ChannelManager::send_probe
                payment_hash: PaymentHash,
                /// The payment path that failed.
-               path: Vec<RouteHop>,
+               path: Path,
                /// The channel responsible for the failed probe.
                ///
                /// Note that for route hints or for the first hop in a path this may be an SCID alias and
@@ -608,12 +665,39 @@ pub enum Event {
                /// The caveat described above the `fee_earned_msat` field applies here as well.
                outbound_amount_forwarded_msat: Option<u64>,
        },
+       /// Used to indicate that a channel with the given `channel_id` is being opened and pending
+       /// confirmation on-chain.
+       ///
+       /// This event is emitted when the funding transaction has been signed and is broadcast to the
+       /// network. For 0conf channels it will be immediately followed by the corresponding
+       /// [`Event::ChannelReady`] event.
+       ChannelPending {
+               /// The `channel_id` of the channel that is pending confirmation.
+               channel_id: [u8; 32],
+               /// The `user_channel_id` value passed in to [`ChannelManager::create_channel`] for outbound
+               /// channels, or to [`ChannelManager::accept_inbound_channel`] for inbound channels if
+               /// [`UserConfig::manually_accept_inbound_channels`] config flag is set to true. Otherwise
+               /// `user_channel_id` will be randomized for an inbound channel.
+               ///
+               /// [`ChannelManager::create_channel`]: crate::ln::channelmanager::ChannelManager::create_channel
+               /// [`ChannelManager::accept_inbound_channel`]: crate::ln::channelmanager::ChannelManager::accept_inbound_channel
+               /// [`UserConfig::manually_accept_inbound_channels`]: crate::util::config::UserConfig::manually_accept_inbound_channels
+               user_channel_id: u128,
+               /// The `temporary_channel_id` this channel used to be known by during channel establishment.
+               ///
+               /// Will be `None` for channels created prior to LDK version 0.0.115.
+               former_temporary_channel_id: Option<[u8; 32]>,
+               /// The `node_id` of the channel counterparty.
+               counterparty_node_id: PublicKey,
+               /// The outpoint of the channel's funding transaction.
+               funding_txo: OutPoint,
+       },
        /// Used to indicate that a channel with the given `channel_id` is ready to
        /// be used. This event is emitted either when the funding transaction has been confirmed
        /// on-chain, or, in case of a 0conf channel, when both parties have confirmed the channel
        /// establishment.
        ChannelReady {
-               /// The channel_id of the channel that is ready.
+               /// The `channel_id` of the channel that is ready.
                channel_id: [u8; 32],
                /// The `user_channel_id` value passed in to [`ChannelManager::create_channel`] for outbound
                /// channels, or to [`ChannelManager::accept_inbound_channel`] for inbound channels if
@@ -624,7 +708,7 @@ pub enum Event {
                /// [`ChannelManager::accept_inbound_channel`]: crate::ln::channelmanager::ChannelManager::accept_inbound_channel
                /// [`UserConfig::manually_accept_inbound_channels`]: crate::util::config::UserConfig::manually_accept_inbound_channels
                user_channel_id: u128,
-               /// The node_id of the channel counterparty.
+               /// The `node_id` of the channel counterparty.
                counterparty_node_id: PublicKey,
                /// The features that this channel will operate with.
                channel_type: ChannelTypeFeatures,
@@ -632,7 +716,7 @@ pub enum Event {
        /// Used to indicate that a previously opened channel with the given `channel_id` is in the
        /// process of closure.
        ChannelClosed  {
-               /// The channel_id of the channel which has been closed. Note that on-chain transactions
+               /// The `channel_id` of the channel which has been closed. Note that on-chain transactions
                /// resolving the channel are likely still awaiting confirmation.
                channel_id: [u8; 32],
                /// The `user_channel_id` value passed in to [`ChannelManager::create_channel`] for outbound
@@ -744,7 +828,10 @@ impl Writeable for Event {
                                // We never write out FundingGenerationReady events as, upon disconnection, peers
                                // drop any channels which have not yet exchanged funding_signed.
                        },
-                       &Event::PaymentClaimable { ref payment_hash, ref amount_msat, ref purpose, ref receiver_node_id, ref via_channel_id, ref via_user_channel_id } => {
+                       &Event::PaymentClaimable { ref payment_hash, ref amount_msat, ref purpose,
+                               ref receiver_node_id, ref via_channel_id, ref via_user_channel_id,
+                               ref claim_deadline, ref onion_fields
+                       } => {
                                1u8.write(writer)?;
                                let mut payment_secret = None;
                                let payment_preimage;
@@ -765,7 +852,9 @@ impl Writeable for Event {
                                        (4, amount_msat, required),
                                        (5, via_user_channel_id, option),
                                        (6, 0u64, required), // user_payment_id required for compatibility with 0.0.103 and earlier
+                                       (7, claim_deadline, option),
                                        (8, payment_preimage, option),
+                                       (9, onion_fields, option),
                                });
                        },
                        &Event::PaymentSent { ref payment_id, ref payment_preimage, ref payment_hash, ref fee_paid_msat } => {
@@ -795,7 +884,8 @@ impl Writeable for Event {
                                        (1, None::<NetworkUpdate>, option), // network_update in LDK versions prior to 0.0.114
                                        (2, payment_failed_permanently, required),
                                        (3, false, required), // all_paths_failed in LDK versions prior to 0.0.114
-                                       (5, *path, vec_type),
+                                       (4, path.blinded_tail, option),
+                                       (5, path.hops, vec_type),
                                        (7, short_channel_id, option),
                                        (9, None::<RouteParameters>, option), // retry in LDK versions prior to 0.0.115
                                        (11, payment_id, option),
@@ -863,13 +953,15 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, option),
-                                       (4, *path, vec_type)
+                                       (4, path.hops, vec_type),
+                                       (6, path.blinded_tail, option),
                                })
                        },
-                       &Event::PaymentFailed { ref payment_id, ref payment_hash } => {
+                       &Event::PaymentFailed { ref payment_id, ref payment_hash, ref reason } => {
                                15u8.write(writer)?;
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
+                                       (1, reason, option),
                                        (2, payment_hash, required),
                                })
                        },
@@ -892,7 +984,8 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, *path, vec_type)
+                                       (4, path.hops, vec_type),
+                                       (6, path.blinded_tail, option),
                                })
                        },
                        &Event::ProbeFailed { ref payment_id, ref payment_hash, ref path, ref short_channel_id } => {
@@ -900,8 +993,9 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, *path, vec_type),
+                                       (4, path.hops, vec_type),
                                        (6, short_channel_id, option),
+                                       (8, path.blinded_tail, option),
                                })
                        },
                        &Event::HTLCHandlingFailed { ref prev_channel_id, ref failed_next_destination } => {
@@ -931,6 +1025,16 @@ impl Writeable for Event {
                                        (6, channel_type, required),
                                });
                        },
+                       &Event::ChannelPending { ref channel_id, ref user_channel_id, ref former_temporary_channel_id, ref counterparty_node_id, ref funding_txo } => {
+                               31u8.write(writer)?;
+                               write_tlv_fields!(writer, {
+                                       (0, channel_id, required),
+                                       (2, user_channel_id, required),
+                                       (4, former_temporary_channel_id, required),
+                                       (6, counterparty_node_id, required),
+                                       (8, funding_txo, required),
+                               });
+                       },
                        // Note that, going forward, all new events must only write data inside of
                        // `write_tlv_fields`. Versions 0.0.101+ will ignore odd-numbered events that write
                        // data via `write_tlv_fields`.
@@ -953,7 +1057,9 @@ impl MaybeReadable for Event {
                                        let mut receiver_node_id = None;
                                        let mut _user_payment_id = None::<u64>; // For compatibility with 0.0.103 and earlier
                                        let mut via_channel_id = None;
+                                       let mut claim_deadline = None;
                                        let mut via_user_channel_id = None;
+                                       let mut onion_fields = None;
                                        read_tlv_fields!(reader, {
                                                (0, payment_hash, required),
                                                (1, receiver_node_id, option),
@@ -962,7 +1068,9 @@ impl MaybeReadable for Event {
                                                (4, amount_msat, required),
                                                (5, via_user_channel_id, option),
                                                (6, _user_payment_id, option),
+                                               (7, claim_deadline, option),
                                                (8, payment_preimage, option),
+                                               (9, onion_fields, option),
                                        });
                                        let purpose = match payment_secret {
                                                Some(secret) => PaymentPurpose::InvoicePayment {
@@ -979,6 +1087,8 @@ impl MaybeReadable for Event {
                                                purpose,
                                                via_channel_id,
                                                via_user_channel_id,
+                                               claim_deadline,
+                                               onion_fields,
                                        }))
                                };
                                f()
@@ -1016,6 +1126,7 @@ impl MaybeReadable for Event {
                                        let mut payment_hash = PaymentHash([0; 32]);
                                        let mut payment_failed_permanently = false;
                                        let mut network_update = None;
+                                       let mut blinded_tail: Option<BlindedTail> = None;
                                        let mut path: Option<Vec<RouteHop>> = Some(vec![]);
                                        let mut short_channel_id = None;
                                        let mut payment_id = None;
@@ -1024,6 +1135,7 @@ impl MaybeReadable for Event {
                                                (0, payment_hash, required),
                                                (1, network_update, upgradable_option),
                                                (2, payment_failed_permanently, required),
+                                               (4, blinded_tail, option),
                                                (5, path, vec_type),
                                                (7, short_channel_id, option),
                                                (11, payment_id, option),
@@ -1035,7 +1147,7 @@ impl MaybeReadable for Event {
                                                payment_hash,
                                                payment_failed_permanently,
                                                failure,
-                                               path: path.unwrap(),
+                                               path: Path { hops: path.unwrap(), blinded_tail },
                                                short_channel_id,
                                                #[cfg(test)]
                                                error_code,
@@ -1138,18 +1250,16 @@ impl MaybeReadable for Event {
                        },
                        13u8 => {
                                let f = || {
-                                       let mut payment_id = PaymentId([0; 32]);
-                                       let mut payment_hash = None;
-                                       let mut path: Option<Vec<RouteHop>> = Some(vec![]);
-                                       read_tlv_fields!(reader, {
+                                       _init_and_read_tlv_fields!(reader, {
                                                (0, payment_id, required),
                                                (2, payment_hash, option),
                                                (4, path, vec_type),
+                                               (6, blinded_tail, option),
                                        });
                                        Ok(Some(Event::PaymentPathSuccessful {
-                                               payment_id,
+                                               payment_id: payment_id.0.unwrap(),
                                                payment_hash,
-                                               path: path.unwrap(),
+                                               path: Path { hops: path.unwrap(), blinded_tail },
                                        }))
                                };
                                f()
@@ -1158,13 +1268,16 @@ impl MaybeReadable for Event {
                                let f = || {
                                        let mut payment_hash = PaymentHash([0; 32]);
                                        let mut payment_id = PaymentId([0; 32]);
+                                       let mut reason = None;
                                        read_tlv_fields!(reader, {
                                                (0, payment_id, required),
+                                               (1, reason, upgradable_option),
                                                (2, payment_hash, required),
                                        });
                                        Ok(Some(Event::PaymentFailed {
                                                payment_id,
                                                payment_hash,
+                                               reason,
                                        }))
                                };
                                f()
@@ -1196,38 +1309,33 @@ impl MaybeReadable for Event {
                        },
                        21u8 => {
                                let f = || {
-                                       let mut payment_id = PaymentId([0; 32]);
-                                       let mut payment_hash = PaymentHash([0; 32]);
-                                       let mut path: Option<Vec<RouteHop>> = Some(vec![]);
-                                       read_tlv_fields!(reader, {
+                                       _init_and_read_tlv_fields!(reader, {
                                                (0, payment_id, required),
                                                (2, payment_hash, required),
                                                (4, path, vec_type),
+                                               (6, blinded_tail, option),
                                        });
                                        Ok(Some(Event::ProbeSuccessful {
-                                               payment_id,
-                                               payment_hash,
-                                               path: path.unwrap(),
+                                               payment_id: payment_id.0.unwrap(),
+                                               payment_hash: payment_hash.0.unwrap(),
+                                               path: Path { hops: path.unwrap(), blinded_tail },
                                        }))
                                };
                                f()
                        },
                        23u8 => {
                                let f = || {
-                                       let mut payment_id = PaymentId([0; 32]);
-                                       let mut payment_hash = PaymentHash([0; 32]);
-                                       let mut path: Option<Vec<RouteHop>> = Some(vec![]);
-                                       let mut short_channel_id = None;
-                                       read_tlv_fields!(reader, {
+                                       _init_and_read_tlv_fields!(reader, {
                                                (0, payment_id, required),
                                                (2, payment_hash, required),
                                                (4, path, vec_type),
                                                (6, short_channel_id, option),
+                                               (8, blinded_tail, option),
                                        });
                                        Ok(Some(Event::ProbeFailed {
-                                               payment_id,
-                                               payment_hash,
-                                               path: path.unwrap(),
+                                               payment_id: payment_id.0.unwrap(),
+                                               payment_hash: payment_hash.0.unwrap(),
+                                               path: Path { hops: path.unwrap(), blinded_tail },
                                                short_channel_id,
                                        }))
                                };
@@ -1271,6 +1379,31 @@ impl MaybeReadable for Event {
                                };
                                f()
                        },
+                       31u8 => {
+                               let f = || {
+                                       let mut channel_id = [0; 32];
+                                       let mut user_channel_id: u128 = 0;
+                                       let mut former_temporary_channel_id = None;
+                                       let mut counterparty_node_id = RequiredWrapper(None);
+                                       let mut funding_txo = RequiredWrapper(None);
+                                       read_tlv_fields!(reader, {
+                                               (0, channel_id, required),
+                                               (2, user_channel_id, required),
+                                               (4, former_temporary_channel_id, required),
+                                               (6, counterparty_node_id, required),
+                                               (8, funding_txo, required),
+                                       });
+
+                                       Ok(Some(Event::ChannelPending {
+                                               channel_id,
+                                               user_channel_id,
+                                               former_temporary_channel_id,
+                                               counterparty_node_id: counterparty_node_id.0.unwrap(),
+                                               funding_txo: funding_txo.0.unwrap()
+                                       }))
+                               };
+                               f()
+                       },
                        // Versions prior to 0.0.100 did not ignore odd types, instead returning InvalidValue.
                        // Version 0.0.100 failed to properly ignore odd types, possibly resulting in corrupt
                        // reads.
index ad16914c35c3476992a1e855e7c8385e9811a7ff..668f752e6c261c455f36be6114f1597c70bb204b 100644 (file)
@@ -81,6 +81,7 @@ pub mod ln;
 pub mod offers;
 pub mod routing;
 pub mod onion_message;
+pub mod blinded_path;
 pub mod events;
 
 #[cfg(feature = "std")]
index d5deb634e1d8468dabed3cc6ee7c9c6edf2d61c8..9699cc78a1a3253a7635277d22c133a3517b2804 100644 (file)
@@ -21,6 +21,7 @@ use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hashes::ripemd160::Hash as Ripemd160;
 use bitcoin::hash_types::{Txid, PubkeyHash};
 
+use crate::chain::keysinterface::EntropySource;
 use crate::ln::{PaymentHash, PaymentPreimage};
 use crate::ln::msgs::DecodeError;
 use crate::util::ser::{Readable, Writeable, Writer};
@@ -39,7 +40,7 @@ use crate::util::transaction_utils::sort_outputs;
 use crate::ln::channel::{INITIAL_COMMITMENT_NUMBER, ANCHOR_OUTPUT_VALUE_SATOSHI};
 use core::ops::Deref;
 use crate::chain;
-use crate::util::crypto::sign;
+use crate::util::crypto::{sign, sign_with_aux_rand};
 
 /// Maximum number of one-way in-flight HTLC (protocol-level value).
 pub const MAX_HTLCS: u16 = 483;
@@ -984,7 +985,7 @@ impl_writeable_tlv_based!(HolderCommitmentTransaction, {
 
 impl HolderCommitmentTransaction {
        #[cfg(test)]
-       pub fn dummy() -> Self {
+       pub fn dummy(htlcs: &mut Vec<(HTLCOutputInCommitment, ())>) -> Self {
                let secp_ctx = Secp256k1::new();
                let dummy_key = PublicKey::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
                let dummy_sig = sign(&secp_ctx, &secp256k1::Message::from_slice(&[42; 32]).unwrap(), &SecretKey::from_slice(&[42; 32]).unwrap());
@@ -1012,12 +1013,16 @@ impl HolderCommitmentTransaction {
                        opt_anchors: None,
                        opt_non_zero_fee_anchors: None,
                };
-               let mut htlcs_with_aux: Vec<(_, ())> = Vec::new();
-               let inner = CommitmentTransaction::new_with_auxiliary_htlc_data(0, 0, 0, false, dummy_key.clone(), dummy_key.clone(), keys, 0, &mut htlcs_with_aux, &channel_parameters.as_counterparty_broadcastable());
+               let mut counterparty_htlc_sigs = Vec::new();
+               for _ in 0..htlcs.len() {
+                       counterparty_htlc_sigs.push(dummy_sig);
+               }
+               let inner = CommitmentTransaction::new_with_auxiliary_htlc_data(0, 0, 0, false, dummy_key.clone(), dummy_key.clone(), keys, 0, htlcs, &channel_parameters.as_counterparty_broadcastable());
+               htlcs.sort_by_key(|htlc| htlc.0.transaction_output_index);
                HolderCommitmentTransaction {
                        inner,
                        counterparty_sig: dummy_sig,
-                       counterparty_htlc_sigs: Vec::new(),
+                       counterparty_htlc_sigs,
                        holder_sig_first: false
                }
        }
@@ -1077,12 +1082,20 @@ impl BuiltCommitmentTransaction {
                hash_to_message!(sighash)
        }
 
-       /// Sign a transaction, either because we are counter-signing the counterparty's transaction or
-       /// because we are about to broadcast a holder transaction.
-       pub fn sign<T: secp256k1::Signing>(&self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64, secp_ctx: &Secp256k1<T>) -> Signature {
+       /// Signs the counterparty's commitment transaction.
+       pub fn sign_counterparty_commitment<T: secp256k1::Signing>(&self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64, secp_ctx: &Secp256k1<T>) -> Signature {
                let sighash = self.get_sighash_all(funding_redeemscript, channel_value_satoshis);
                sign(secp_ctx, &sighash, funding_key)
        }
+
+       /// Signs the holder commitment transaction because we are about to broadcast it.
+       pub fn sign_holder_commitment<T: secp256k1::Signing, ES: Deref>(
+               &self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64,
+               entropy_source: &ES, secp_ctx: &Secp256k1<T>
+       ) -> Signature where ES::Target: EntropySource {
+               let sighash = self.get_sighash_all(funding_redeemscript, channel_value_satoshis);
+               sign_with_aux_rand(secp_ctx, &sighash, funding_key, entropy_source)
+       }
 }
 
 /// This class tracks the per-transaction information needed to build a closing transaction and will
@@ -1559,7 +1572,10 @@ impl<'a> TrustedCommitmentTransaction<'a> {
        /// The returned Vec has one entry for each HTLC, and in the same order.
        ///
        /// This function is only valid in the holder commitment context, it always uses EcdsaSighashType::All.
-       pub fn get_htlc_sigs<T: secp256k1::Signing>(&self, htlc_base_key: &SecretKey, channel_parameters: &DirectedChannelTransactionParameters, secp_ctx: &Secp256k1<T>) -> Result<Vec<Signature>, ()> {
+       pub fn get_htlc_sigs<T: secp256k1::Signing, ES: Deref>(
+               &self, htlc_base_key: &SecretKey, channel_parameters: &DirectedChannelTransactionParameters,
+               entropy_source: &ES, secp_ctx: &Secp256k1<T>,
+       ) -> Result<Vec<Signature>, ()> where ES::Target: EntropySource {
                let inner = self.inner;
                let keys = &inner.keys;
                let txid = inner.built.txid;
@@ -1573,7 +1589,7 @@ impl<'a> TrustedCommitmentTransaction<'a> {
                        let htlc_redeemscript = get_htlc_redeemscript_with_explicit_keys(&this_htlc, self.opt_anchors(), &keys.broadcaster_htlc_key, &keys.countersignatory_htlc_key, &keys.revocation_key);
 
                        let sighash = hash_to_message!(&sighash::SighashCache::new(&htlc_tx).segwit_signature_hash(0, &htlc_redeemscript, this_htlc.amount_msat / 1000, EcdsaSighashType::All).unwrap()[..]);
-                       ret.push(sign(secp_ctx, &sighash, &holder_htlc_key));
+                       ret.push(sign_with_aux_rand(secp_ctx, &sighash, &holder_htlc_key, entropy_source));
                }
                Ok(ret)
        }
index b57f70d7901d21aa57c5ebe954ad21ca92a276a8..3f210cb46c535dfb433575d558740882f4446cd3 100644 (file)
@@ -20,7 +20,7 @@ use crate::chain::channelmonitor::{ANTI_REORG_DELAY, ChannelMonitor};
 use crate::chain::transaction::OutPoint;
 use crate::chain::{ChannelMonitorUpdateStatus, Listen, Watch};
 use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason, HTLCDestination};
-use crate::ln::channelmanager::{ChannelManager, RAACommitmentOrder, PaymentSendFailure, PaymentId};
+use crate::ln::channelmanager::{ChannelManager, RAACommitmentOrder, PaymentSendFailure, PaymentId, RecipientOnionFields};
 use crate::ln::channel::AnnouncementSigsState;
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler};
@@ -50,7 +50,9 @@ fn test_simple_monitor_permanent_update_fail() {
 
        let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(&nodes[0], nodes[1], 1000000);
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
-       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)
+               ), true, APIError::ChannelUnavailable {..}, {});
        check_added_monitors!(nodes[0], 2);
 
        let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
@@ -173,7 +175,9 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 
        {
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                               RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)
+                       ), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -204,7 +208,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        let events_3 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_3.len(), 1);
        match events_3[0] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(payment_hash_1, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
                        assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
@@ -226,7 +230,9 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(&nodes[0], nodes[1], 1000000);
        {
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)
+                       ), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -290,7 +296,9 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)
+                       ), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -573,7 +581,7 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
        let events_5 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_5.len(), 1);
        match events_5[0] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(payment_hash_2, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
                        assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
@@ -630,7 +638,8 @@ fn test_monitor_update_fail_cs() {
 
        let (route, our_payment_hash, payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -690,7 +699,7 @@ fn test_monitor_update_fail_cs() {
        let events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
-               Event::PaymentClaimable { payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(payment_hash, our_payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
                        assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
@@ -722,7 +731,8 @@ fn test_monitor_update_fail_no_rebroadcast() {
 
        let (route, our_payment_hash, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(payment_secret_1), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -769,14 +779,16 @@ fn test_monitor_update_raa_while_paused() {
        send_payment(&nodes[0], &[&nodes[1]], 5000000);
        let (route, our_payment_hash_1, payment_preimage_1, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash_1,
+                       RecipientOnionFields::secret_only(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
        let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 
        let (route, our_payment_hash_2, payment_preimage_2, our_payment_secret_2) = get_route_and_payment_hash!(nodes[1], nodes[0], 1000000);
        {
-               nodes[1].node.send_payment(&route, our_payment_hash_2, &Some(our_payment_secret_2), PaymentId(our_payment_hash_2.0)).unwrap();
+               nodes[1].node.send_payment_with_route(&route, our_payment_hash_2,
+                       RecipientOnionFields::secret_only(our_payment_secret_2), PaymentId(our_payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
        }
        let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
@@ -864,7 +876,8 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        // holding cell.
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -888,7 +901,8 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        // being paused waiting a monitor update.
        let (route, payment_hash_3, _, payment_secret_3) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_3,
+                       RecipientOnionFields::secret_only(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -907,7 +921,8 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
                // Try to route another payment backwards from 2 to make sure 1 holds off on responding
                let (route, payment_hash_4, payment_preimage_4, payment_secret_4) = get_route_and_payment_hash!(nodes[2], nodes[0], 1000000);
-               nodes[2].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
+               nodes[2].node.send_payment_with_route(&route, payment_hash_4,
+                       RecipientOnionFields::secret_only(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
 
                send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
@@ -1205,9 +1220,11 @@ fn raa_no_response_awaiting_raa_state() {
        // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
        // generation during RAA while in monitor-update-failed state.
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
        }
 
@@ -1256,7 +1273,8 @@ fn raa_no_response_awaiting_raa_state() {
        // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
        // commitment transaction states) whereas here we can explicitly check for it.
        {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_3,
+                       RecipientOnionFields::secret_only(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        }
@@ -1345,7 +1363,8 @@ fn claim_while_disconnected_monitor_update_fail() {
        // the monitor still failed
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1439,7 +1458,8 @@ fn monitor_failed_no_reestablish_response() {
        // on receipt).
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1511,7 +1531,8 @@ fn first_message_on_recv_ordering() {
        // can deliver it and fail the monitor update.
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1534,7 +1555,8 @@ fn first_message_on_recv_ordering() {
        // Route the second payment, generating an update_add_htlc/commitment_signed
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1615,7 +1637,8 @@ fn test_monitor_update_fail_claim() {
 
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[2], nodes[0], 1_000_000);
        {
-               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[2].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
 
@@ -1633,7 +1656,8 @@ fn test_monitor_update_fail_claim() {
        expect_pending_htlcs_forwardable_ignore!(nodes[1]);
 
        let (_, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[0]);
-       nodes[2].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
+       nodes[2].node.send_payment_with_route(&route, payment_hash_3,
+               RecipientOnionFields::secret_only(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
        check_added_monitors!(nodes[2], 1);
 
        let mut events = nodes[2].node.get_and_clear_pending_msg_events();
@@ -1668,7 +1692,7 @@ fn test_monitor_update_fail_claim() {
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
        match events[0] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id, .. } => {
                        assert_eq!(payment_hash_2, *payment_hash);
                        assert_eq!(1_000_000, amount_msat);
                        assert_eq!(receiver_node_id.unwrap(), nodes[0].node.get_our_node_id());
@@ -1685,7 +1709,7 @@ fn test_monitor_update_fail_claim() {
                _ => panic!("Unexpected event"),
        }
        match events[1] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(payment_hash_3, *payment_hash);
                        assert_eq!(1_000_000, amount_msat);
                        assert_eq!(receiver_node_id.unwrap(), nodes[0].node.get_our_node_id());
@@ -1730,7 +1754,8 @@ fn test_monitor_update_on_pending_forwards() {
 
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[2], nodes[0], 1000000);
        {
-               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[2].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
 
@@ -1794,7 +1819,8 @@ fn monitor_update_claim_fail_no_response() {
        // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1864,6 +1890,7 @@ fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf:
        let (outpoint, latest_update, _) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
        nodes[0].chain_monitor.chain_monitor.force_channel_monitor_updated(outpoint, latest_update);
        check_added_monitors!(nodes[0], 0);
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
 
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 0);
@@ -1885,6 +1912,9 @@ fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf:
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
        reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       // But we want to re-emit ChannelPending
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
@@ -1970,12 +2000,12 @@ fn test_path_paused_mpp() {
        // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
        let path = route.paths[0].clone();
        route.paths.push(path);
-       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-       route.paths[0][0].short_channel_id = chan_1_id;
-       route.paths[0][1].short_channel_id = chan_3_id;
-       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-       route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
-       route.paths[1][1].short_channel_id = chan_4_id;
+       route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0].hops[0].short_channel_id = chan_1_id;
+       route.paths[0].hops[1].short_channel_id = chan_3_id;
+       route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1].hops[0].short_channel_id = chan_2_ann.contents.short_channel_id;
+       route.paths[1].hops[1].short_channel_id = chan_4_id;
 
        // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
        // (for the path 0 -> 2 -> 3) fails.
@@ -1985,7 +2015,9 @@ fn test_path_paused_mpp() {
        // Now check that we get the right return value, indicating that the first path succeeded but
        // the second got a MonitorUpdateInProgress err. This implies
        // PaymentSendFailure::PartialFailure as some paths succeeded, preventing retry.
-       if let Err(PaymentSendFailure::PartialFailure { results, ..}) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) {
+       if let Err(PaymentSendFailure::PartialFailure { results, ..}) = nodes[0].node.send_payment_with_route(
+               &route, payment_hash, RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+       ) {
                assert_eq!(results.len(), 2);
                if let Ok(()) = results[0] {} else { panic!(); }
                if let Err(APIError::MonitorUpdateInProgress) = results[1] {} else { panic!(); }
@@ -2030,7 +2062,8 @@ fn test_pending_update_fee_ack_on_reconnect() {
        send_payment(&nodes[0], &[&nodes[1]], 100_000_00);
 
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[1], nodes[0], 1_000_000);
-       nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
        let bs_initial_send_msgs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        // bs_initial_send_msgs are not delivered until they are re-generated after reconnect
@@ -2281,12 +2314,14 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
        // (c) will not be freed from the holding cell.
        let (payment_preimage_0, payment_hash_0, _) = route_payment(&nodes[1], &[&nodes[0]], 100_000);
 
-       nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+               RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let send = SendEvent::from_node(&nodes[0]);
        assert_eq!(send.msgs.len(), 1);
 
-       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
 
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
@@ -2447,7 +2482,8 @@ fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_f
                // In order to get the HTLC claim into the holding cell at nodes[1], we need nodes[1] to be
                // awaiting a remote revoke_and_ack from nodes[0].
                let (route, second_payment_hash, _, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+                       RecipientOnionFields::secret_only(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
@@ -2808,6 +2844,7 @@ fn do_test_outbound_reload_without_init_mon(use_0conf: bool) {
        let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
        nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
        check_added_monitors!(nodes[1], 1);
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        let bs_signed_locked = nodes[1].node.get_and_clear_pending_msg_events();
        assert_eq!(bs_signed_locked.len(), if use_0conf { 2 } else { 1 });
@@ -2906,6 +2943,7 @@ fn do_test_inbound_reload_without_init_mon(use_0conf: bool, lock_commitment: boo
 
        nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
        check_added_monitors!(nodes[0], 1);
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
 
        let as_funding_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        if lock_commitment {
index f1c5cae25ae8bbfbc84b85640c3367de18aa56c1..8b5e89d9441ace4510bddd65f877fc7355f94604 100644 (file)
@@ -312,9 +312,9 @@ pub(super) enum ChannelUpdateStatus {
        /// We've announced the channel as enabled and are connected to our peer.
        Enabled,
        /// Our channel is no longer live, but we haven't announced the channel as disabled yet.
-       DisabledStaged,
+       DisabledStaged(u8),
        /// Our channel is live again, but we haven't announced the channel as enabled yet.
-       EnabledStaged,
+       EnabledStaged(u8),
        /// We've announced the channel as disabled.
        Disabled,
 }
@@ -499,6 +499,7 @@ pub(super) struct Channel<Signer: ChannelSigner> {
        user_id: u128,
 
        channel_id: [u8; 32],
+       temporary_channel_id: Option<[u8; 32]>, // Will be `None` for channels created prior to 0.0.115.
        channel_state: u32,
 
        // When we reach max(6 blocks, minimum_depth), we need to send an AnnouncementSigs message to
@@ -652,7 +653,7 @@ pub(super) struct Channel<Signer: ChannelSigner> {
        pub counterparty_max_accepted_htlcs: u16,
        #[cfg(not(test))]
        counterparty_max_accepted_htlcs: u16,
-       //implied by OUR_MAX_HTLCS: max_accepted_htlcs: u16,
+       holder_max_accepted_htlcs: u16,
        minimum_depth: Option<u32>,
 
        counterparty_forwarding_info: Option<CounterpartyForwardingInfo>,
@@ -729,6 +730,9 @@ pub(super) struct Channel<Signer: ChannelSigner> {
        // blinded paths instead of simple scid+node_id aliases.
        outbound_scid_alias: u64,
 
+       // We track whether we already emitted a `ChannelPending` event.
+       channel_pending_event_emitted: bool,
+
        // We track whether we already emitted a `ChannelReady` event.
        channel_ready_event_emitted: bool,
 
@@ -752,7 +756,7 @@ struct CommitmentTxInfoCached {
        feerate: u32,
 }
 
-pub const OUR_MAX_HTLCS: u16 = 50; //TODO
+pub const DEFAULT_MAX_HTLCS: u16 = 50;
 
 pub(crate) fn commitment_tx_base_weight(opt_anchors: bool) -> u64 {
        const COMMITMENT_TX_BASE_WEIGHT: u64 = 724;
@@ -991,6 +995,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        }
                }
 
+               let temporary_channel_id = entropy_source.get_secure_random_bytes();
+
                Ok(Channel {
                        user_id,
 
@@ -1004,7 +1010,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
 
                        inbound_handshake_limits_override: Some(config.channel_handshake_limits.clone()),
 
-                       channel_id: entropy_source.get_secure_random_bytes(),
+                       channel_id: temporary_channel_id,
+                       temporary_channel_id: Some(temporary_channel_id),
                        channel_state: ChannelState::OurInitSent as u32,
                        announcement_sigs_state: AnnouncementSigsState::NotSent,
                        secp_ctx,
@@ -1065,6 +1072,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        counterparty_htlc_minimum_msat: 0,
                        holder_htlc_minimum_msat: if config.channel_handshake_config.our_htlc_minimum_msat == 0 { 1 } else { config.channel_handshake_config.our_htlc_minimum_msat },
                        counterparty_max_accepted_htlcs: 0,
+                       holder_max_accepted_htlcs: cmp::min(config.channel_handshake_config.our_max_accepted_htlcs, MAX_HTLCS),
                        minimum_depth: None, // Filled in in accept_channel
 
                        counterparty_forwarding_info: None,
@@ -1103,6 +1111,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        latest_inbound_scid_alias: None,
                        outbound_scid_alias,
 
+                       channel_pending_event_emitted: false,
                        channel_ready_event_emitted: false,
 
                        #[cfg(any(test, fuzzing))]
@@ -1350,6 +1359,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        inbound_handshake_limits_override: None,
 
                        channel_id: msg.temporary_channel_id,
+                       temporary_channel_id: Some(msg.temporary_channel_id),
                        channel_state: (ChannelState::OurInitSent as u32) | (ChannelState::TheirInitSent as u32),
                        announcement_sigs_state: AnnouncementSigsState::NotSent,
                        secp_ctx,
@@ -1410,6 +1420,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        counterparty_htlc_minimum_msat: msg.htlc_minimum_msat,
                        holder_htlc_minimum_msat: if config.channel_handshake_config.our_htlc_minimum_msat == 0 { 1 } else { config.channel_handshake_config.our_htlc_minimum_msat },
                        counterparty_max_accepted_htlcs: msg.max_accepted_htlcs,
+                       holder_max_accepted_htlcs: cmp::min(config.channel_handshake_config.our_max_accepted_htlcs, MAX_HTLCS),
                        minimum_depth: Some(cmp::max(config.channel_handshake_config.minimum_depth, 1)),
 
                        counterparty_forwarding_info: None,
@@ -1451,6 +1462,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        latest_inbound_scid_alias: None,
                        outbound_scid_alias,
 
+                       channel_pending_event_emitted: false,
                        channel_ready_event_emitted: false,
 
                        #[cfg(any(test, fuzzing))]
@@ -2352,7 +2364,9 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
 
                Ok((msgs::FundingSigned {
                        channel_id: self.channel_id,
-                       signature
+                       signature,
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
                }, channel_monitor))
        }
 
@@ -2862,8 +2876,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
 
                let inbound_stats = self.get_inbound_pending_htlc_stats(None);
                let outbound_stats = self.get_outbound_pending_htlc_stats(None);
-               if inbound_stats.pending_htlcs + 1 > OUR_MAX_HTLCS as u32 {
-                       return Err(ChannelError::Close(format!("Remote tried to push more than our max accepted HTLCs ({})", OUR_MAX_HTLCS)));
+               if inbound_stats.pending_htlcs + 1 > self.holder_max_accepted_htlcs as u32 {
+                       return Err(ChannelError::Close(format!("Remote tried to push more than our max accepted HTLCs ({})", self.holder_max_accepted_htlcs)));
                }
                if inbound_stats.pending_htlcs_value_msat + msg.amount_msat > self.holder_max_htlc_value_in_flight_msat {
                        return Err(ChannelError::Close(format!("Remote HTLC add would put them over our max HTLC value ({})", self.holder_max_htlc_value_in_flight_msat)));
@@ -3122,9 +3136,24 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        return Err(ChannelError::Close(format!("Got wrong number of HTLC signatures ({}) from remote. It must be {}", msg.htlc_signatures.len(), commitment_stats.num_nondust_htlcs)));
                }
 
-               // TODO: Sadly, we pass HTLCs twice to ChannelMonitor: once via the HolderCommitmentTransaction and once via the update
+               // Up to LDK 0.0.115, HTLC information was required to be duplicated in the
+               // `htlcs_and_sigs` vec and in the `holder_commitment_tx` itself, both of which were passed
+               // in the `ChannelMonitorUpdate`. In 0.0.115, support for having a separate set of
+               // outbound-non-dust-HTLCSources in the `ChannelMonitorUpdate` was added, however for
+               // backwards compatibility, we never use it in production. To provide test coverage, here,
+               // we randomly decide (in test/fuzzing builds) to use the new vec sometimes.
+               #[allow(unused_assignments, unused_mut)]
+               let mut separate_nondust_htlc_sources = false;
+               #[cfg(all(feature = "std", any(test, fuzzing)))] {
+                       use core::hash::{BuildHasher, Hasher};
+                       // Get a random value using the only std API to do so - the DefaultHasher
+                       let rand_val = std::collections::hash_map::RandomState::new().build_hasher().finish();
+                       separate_nondust_htlc_sources = rand_val % 2 == 0;
+               }
+
+               let mut nondust_htlc_sources = Vec::with_capacity(htlcs_cloned.len());
                let mut htlcs_and_sigs = Vec::with_capacity(htlcs_cloned.len());
-               for (idx, (htlc, source)) in htlcs_cloned.drain(..).enumerate() {
+               for (idx, (htlc, mut source_opt)) in htlcs_cloned.drain(..).enumerate() {
                        if let Some(_) = htlc.transaction_output_index {
                                let htlc_tx = chan_utils::build_htlc_transaction(&commitment_txid, commitment_stats.feerate_per_kw,
                                        self.get_counterparty_selected_contest_delay().unwrap(), &htlc, self.opt_anchors(),
@@ -3139,10 +3168,18 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                                if let Err(_) = self.secp_ctx.verify_ecdsa(&htlc_sighash, &msg.htlc_signatures[idx], &keys.countersignatory_htlc_key) {
                                        return Err(ChannelError::Close("Invalid HTLC tx signature from peer".to_owned()));
                                }
-                               htlcs_and_sigs.push((htlc, Some(msg.htlc_signatures[idx]), source));
+                               if !separate_nondust_htlc_sources {
+                                       htlcs_and_sigs.push((htlc, Some(msg.htlc_signatures[idx]), source_opt.take()));
+                               }
                        } else {
-                               htlcs_and_sigs.push((htlc, None, source));
+                               htlcs_and_sigs.push((htlc, None, source_opt.take()));
                        }
+                       if separate_nondust_htlc_sources {
+                               if let Some(source) = source_opt.take() {
+                                       nondust_htlc_sources.push(source);
+                               }
+                       }
+                       debug_assert!(source_opt.is_none(), "HTLCSource should have been put somewhere");
                }
 
                let holder_commitment_tx = HolderCommitmentTransaction::new(
@@ -3205,6 +3242,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                                commitment_tx: holder_commitment_tx,
                                htlc_outputs: htlcs_and_sigs,
                                claimed_htlcs,
+                               nondust_htlc_sources,
                        }]
                };
 
@@ -3908,6 +3946,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        channel_id: self.channel_id,
                        per_commitment_secret,
                        next_per_commitment_point,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                }
        }
 
@@ -4550,6 +4590,13 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                self.channel_id
        }
 
+       // Return the `temporary_channel_id` used during channel establishment.
+       //
+       // Will return `None` for channels created prior to LDK version 0.0.115.
+       pub fn temporary_channel_id(&self) -> Option<[u8; 32]> {
+               self.temporary_channel_id
+       }
+
        pub fn minimum_depth(&self) -> Option<u32> {
                self.minimum_depth
        }
@@ -4694,6 +4741,21 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                self.prev_config.map(|prev_config| prev_config.0)
        }
 
+       // Checks whether we should emit a `ChannelPending` event.
+       pub(crate) fn should_emit_channel_pending_event(&mut self) -> bool {
+               self.is_funding_initiated() && !self.channel_pending_event_emitted
+       }
+
+       // Returns whether we already emitted a `ChannelPending` event.
+       pub(crate) fn channel_pending_event_emitted(&self) -> bool {
+               self.channel_pending_event_emitted
+       }
+
+       // Remembers that we already emitted a `ChannelPending` event.
+       pub(crate) fn set_channel_pending_event_emitted(&mut self) {
+               self.channel_pending_event_emitted = true;
+       }
+
        // Checks whether we should emit a `ChannelReady` event.
        pub(crate) fn should_emit_channel_ready_event(&mut self) -> bool {
                self.is_usable() && !self.channel_ready_event_emitted
@@ -5253,7 +5315,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        htlc_minimum_msat: self.holder_htlc_minimum_msat,
                        feerate_per_kw: self.feerate_per_kw as u32,
                        to_self_delay: self.get_holder_selected_contest_delay(),
-                       max_accepted_htlcs: OUR_MAX_HTLCS,
+                       max_accepted_htlcs: self.holder_max_accepted_htlcs,
                        funding_pubkey: keys.funding_pubkey,
                        revocation_basepoint: keys.revocation_basepoint,
                        payment_point: keys.payment_point,
@@ -5320,7 +5382,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        htlc_minimum_msat: self.holder_htlc_minimum_msat,
                        minimum_depth: self.minimum_depth.unwrap(),
                        to_self_delay: self.get_holder_selected_contest_delay(),
-                       max_accepted_htlcs: OUR_MAX_HTLCS,
+                       max_accepted_htlcs: self.holder_max_accepted_htlcs,
                        funding_pubkey: keys.funding_pubkey,
                        revocation_basepoint: keys.revocation_basepoint,
                        payment_point: keys.payment_point,
@@ -5332,6 +5394,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                                None => Builder::new().into_script(),
                        }),
                        channel_type: Some(self.channel_type.clone()),
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                }
        }
 
@@ -5396,7 +5460,11 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        temporary_channel_id,
                        funding_txid: funding_txo.txid,
                        funding_output_index: funding_txo.index,
-                       signature
+                       signature,
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                })
        }
 
@@ -5915,6 +5983,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Channel<Signer> {
                        channel_id: self.channel_id,
                        signature,
                        htlc_signatures,
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
                }, (counterparty_commitment_txid, commitment_stats.htlcs_included)))
        }
 
@@ -6123,8 +6193,8 @@ impl Writeable for ChannelUpdateStatus {
                // channel as enabled, so we write 0. For EnabledStaged, we similarly write a 1.
                match self {
                        ChannelUpdateStatus::Enabled => 0u8.write(writer)?,
-                       ChannelUpdateStatus::DisabledStaged => 0u8.write(writer)?,
-                       ChannelUpdateStatus::EnabledStaged => 1u8.write(writer)?,
+                       ChannelUpdateStatus::DisabledStaged(_) => 0u8.write(writer)?,
+                       ChannelUpdateStatus::EnabledStaged(_) => 1u8.write(writer)?,
                        ChannelUpdateStatus::Disabled => 1u8.write(writer)?,
                }
                Ok(())
@@ -6420,6 +6490,7 @@ impl<Signer: WriteableEcdsaChannelSigner> Writeable for Channel<Signer> {
                        if self.holder_max_htlc_value_in_flight_msat != Self::get_holder_max_htlc_value_in_flight_msat(self.channel_value_satoshis, &old_max_in_flight_percent_config)
                        { Some(self.holder_max_htlc_value_in_flight_msat) } else { None };
 
+               let channel_pending_event_emitted = Some(self.channel_pending_event_emitted);
                let channel_ready_event_emitted = Some(self.channel_ready_event_emitted);
 
                // `user_id` used to be a single u64 value. In order to remain backwards compatible with
@@ -6427,6 +6498,8 @@ impl<Signer: WriteableEcdsaChannelSigner> Writeable for Channel<Signer> {
                // we write the high bytes as an option here.
                let user_id_high_opt = Some((self.user_id >> 64) as u64);
 
+               let holder_max_accepted_htlcs = if self.holder_max_accepted_htlcs == DEFAULT_MAX_HTLCS { None } else { Some(self.holder_max_accepted_htlcs) };
+
                write_tlv_fields!(writer, {
                        (0, self.announcement_sigs, option),
                        // minimum_depth and counterparty_selected_channel_reserve_satoshis used to have a
@@ -6452,6 +6525,9 @@ impl<Signer: WriteableEcdsaChannelSigner> Writeable for Channel<Signer> {
                        (23, channel_ready_event_emitted, option),
                        (25, user_id_high_opt, option),
                        (27, self.channel_keys_id, required),
+                       (28, holder_max_accepted_htlcs, option),
+                       (29, self.temporary_channel_id, option),
+                       (31, channel_pending_event_emitted, option),
                });
 
                Ok(())
@@ -6518,7 +6594,8 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                let value_to_self_msat = Readable::read(reader)?;
 
                let pending_inbound_htlc_count: u64 = Readable::read(reader)?;
-               let mut pending_inbound_htlcs = Vec::with_capacity(cmp::min(pending_inbound_htlc_count as usize, OUR_MAX_HTLCS as usize));
+
+               let mut pending_inbound_htlcs = Vec::with_capacity(cmp::min(pending_inbound_htlc_count as usize, DEFAULT_MAX_HTLCS as usize));
                for _ in 0..pending_inbound_htlc_count {
                        pending_inbound_htlcs.push(InboundHTLCOutput {
                                htlc_id: Readable::read(reader)?,
@@ -6536,7 +6613,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                }
 
                let pending_outbound_htlc_count: u64 = Readable::read(reader)?;
-               let mut pending_outbound_htlcs = Vec::with_capacity(cmp::min(pending_outbound_htlc_count as usize, OUR_MAX_HTLCS as usize));
+               let mut pending_outbound_htlcs = Vec::with_capacity(cmp::min(pending_outbound_htlc_count as usize, DEFAULT_MAX_HTLCS as usize));
                for _ in 0..pending_outbound_htlc_count {
                        pending_outbound_htlcs.push(OutboundHTLCOutput {
                                htlc_id: Readable::read(reader)?,
@@ -6565,7 +6642,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                }
 
                let holding_cell_htlc_update_count: u64 = Readable::read(reader)?;
-               let mut holding_cell_htlc_updates = Vec::with_capacity(cmp::min(holding_cell_htlc_update_count as usize, OUR_MAX_HTLCS as usize*2));
+               let mut holding_cell_htlc_updates = Vec::with_capacity(cmp::min(holding_cell_htlc_update_count as usize, DEFAULT_MAX_HTLCS as usize*2));
                for _ in 0..holding_cell_htlc_update_count {
                        holding_cell_htlc_updates.push(match <u8 as Readable>::read(reader)? {
                                0 => HTLCUpdateAwaitingACK::AddHTLC {
@@ -6598,13 +6675,13 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                let monitor_pending_commitment_signed = Readable::read(reader)?;
 
                let monitor_pending_forwards_count: u64 = Readable::read(reader)?;
-               let mut monitor_pending_forwards = Vec::with_capacity(cmp::min(monitor_pending_forwards_count as usize, OUR_MAX_HTLCS as usize));
+               let mut monitor_pending_forwards = Vec::with_capacity(cmp::min(monitor_pending_forwards_count as usize, DEFAULT_MAX_HTLCS as usize));
                for _ in 0..monitor_pending_forwards_count {
                        monitor_pending_forwards.push((Readable::read(reader)?, Readable::read(reader)?));
                }
 
                let monitor_pending_failures_count: u64 = Readable::read(reader)?;
-               let mut monitor_pending_failures = Vec::with_capacity(cmp::min(monitor_pending_failures_count as usize, OUR_MAX_HTLCS as usize));
+               let mut monitor_pending_failures = Vec::with_capacity(cmp::min(monitor_pending_failures_count as usize, DEFAULT_MAX_HTLCS as usize));
                for _ in 0..monitor_pending_failures_count {
                        monitor_pending_failures.push((Readable::read(reader)?, Readable::read(reader)?, Readable::read(reader)?));
                }
@@ -6719,10 +6796,13 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                let mut announcement_sigs_state = Some(AnnouncementSigsState::NotSent);
                let mut latest_inbound_scid_alias = None;
                let mut outbound_scid_alias = None;
+               let mut channel_pending_event_emitted = None;
                let mut channel_ready_event_emitted = None;
 
                let mut user_id_high_opt: Option<u64> = None;
                let mut channel_keys_id: Option<[u8; 32]> = None;
+               let mut temporary_channel_id: Option<[u8; 32]> = None;
+               let mut holder_max_accepted_htlcs: Option<u16> = None;
 
                read_tlv_fields!(reader, {
                        (0, announcement_sigs, option),
@@ -6743,6 +6823,9 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                        (23, channel_ready_event_emitted, option),
                        (25, user_id_high_opt, option),
                        (27, channel_keys_id, option),
+                       (28, holder_max_accepted_htlcs, option),
+                       (29, temporary_channel_id, option),
+                       (31, channel_pending_event_emitted, option),
                });
 
                let (channel_keys_id, holder_signer) = if let Some(channel_keys_id) = channel_keys_id {
@@ -6795,6 +6878,8 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                // separate u64 values.
                let user_id = user_id_low as u128 + ((user_id_high_opt.unwrap_or(0) as u128) << 64);
 
+               let holder_max_accepted_htlcs = holder_max_accepted_htlcs.unwrap_or(DEFAULT_MAX_HTLCS);
+
                Ok(Channel {
                        user_id,
 
@@ -6807,6 +6892,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                        inbound_handshake_limits_override: None,
 
                        channel_id,
+                       temporary_channel_id,
                        channel_state,
                        announcement_sigs_state: announcement_sigs_state.unwrap(),
                        secp_ctx,
@@ -6822,6 +6908,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                        cur_counterparty_commitment_transaction_number,
                        value_to_self_msat,
 
+                       holder_max_accepted_htlcs,
                        pending_inbound_htlcs,
                        pending_outbound_htlcs,
                        holding_cell_htlc_updates,
@@ -6899,6 +6986,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch
                        // Later in the ChannelManager deserialization phase we scan for channels and assign scid aliases if its missing
                        outbound_scid_alias: outbound_scid_alias.unwrap_or(0),
 
+                       channel_pending_event_emitted: channel_pending_event_emitted.unwrap_or(true),
                        channel_ready_event_emitted: channel_ready_event_emitted.unwrap_or(true),
 
                        #[cfg(any(test, fuzzing))]
@@ -6936,6 +7024,7 @@ mod tests {
        use crate::chain::chaininterface::{FeeEstimator, LowerBoundedFeeEstimator, ConfirmationTarget};
        use crate::chain::keysinterface::{ChannelSigner, InMemorySigner, EntropySource, SignerProvider};
        use crate::chain::transaction::OutPoint;
+       use crate::routing::router::Path;
        use crate::util::config::UserConfig;
        use crate::util::enforcing_trait_impls::EnforcingSigner;
        use crate::util::errors::APIError;
@@ -7113,11 +7202,10 @@ mod tests {
                        cltv_expiry: 200000000,
                        state: OutboundHTLCState::Committed,
                        source: HTLCSource::OutboundRoute {
-                               path: Vec::new(),
+                               path: Path { hops: Vec::new(), blinded_tail: None },
                                session_priv: SecretKey::from_slice(&hex::decode("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap()[..]).unwrap(),
                                first_hop_htlc_msat: 548,
                                payment_id: PaymentId([42; 32]),
-                               payment_secret: None,
                        }
                });
 
@@ -7429,7 +7517,7 @@ mod tests {
                }
        }
 
-       #[cfg(not(feature = "grind_signatures"))]
+       #[cfg(feature = "_test_vectors")]
        #[test]
        fn outbound_commitment_test() {
                use bitcoin::util::sighash;
@@ -7462,6 +7550,7 @@ mod tests {
                        [0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
                        10_000_000,
                        [0; 32],
+                       [0; 32],
                );
 
                assert_eq!(signer.pubkeys().funding_pubkey.serialize()[..],
index d591788f64a68209349b2d7b86c269997a63fbab..498813de7a11900e2dbdcffa9adf8de9fd037d6c 100644 (file)
@@ -36,7 +36,7 @@ use crate::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, Fee
 use crate::chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateStep, HTLC_FAIL_BACK_BUFFER, CLTV_CLAIM_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS, ANTI_REORG_DELAY, MonitorEvent, CLOSED_CHANNEL_UPDATE_ID};
 use crate::chain::transaction::{OutPoint, TransactionData};
 use crate::events;
-use crate::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
+use crate::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination, PaymentFailureReason};
 // Since this struct is returned in `list_channels` methods, expose it here in case users want to
 // construct one themselves.
 use crate::ln::{inbound_payment, PaymentHash, PaymentPreimage, PaymentSecret};
@@ -45,7 +45,7 @@ use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, No
 #[cfg(any(feature = "_test_utils", test))]
 use crate::ln::features::InvoiceFeatures;
 use crate::routing::gossip::NetworkGraph;
-use crate::routing::router::{DefaultRouter, InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, RoutePath, Router};
+use crate::routing::router::{BlindedTail, DefaultRouter, InFlightHtlcs, Path, PaymentParameters, Route, RouteHop, RouteParameters, Router};
 use crate::routing::scoring::ProbabilisticScorer;
 use crate::ln::msgs;
 use crate::ln::onion_utils;
@@ -72,12 +72,12 @@ use core::{cmp, mem};
 use core::cell::RefCell;
 use crate::io::Read;
 use crate::sync::{Arc, Mutex, RwLock, RwLockReadGuard, FairRwLock, LockTestExt, LockHeldState};
-use core::sync::atomic::{AtomicUsize, Ordering};
+use core::sync::atomic::{AtomicUsize, AtomicBool, Ordering};
 use core::time::Duration;
 use core::ops::Deref;
 
 // Re-export this for use in the public API.
-pub use crate::ln::outbound_payment::{PaymentSendFailure, Retry, RetryableSendFailure};
+pub use crate::ln::outbound_payment::{PaymentSendFailure, Retry, RetryableSendFailure, RecipientOnionFields};
 
 // We hold various information about HTLC relay in the HTLC objects in Channel itself:
 //
@@ -106,11 +106,13 @@ pub(super) enum PendingHTLCRouting {
        },
        Receive {
                payment_data: msgs::FinalOnionHopData,
+               payment_metadata: Option<Vec<u8>>,
                incoming_cltv_expiry: u32, // Used to track when we should expire pending HTLCs that go unclaimed
                phantom_shared_secret: Option<[u8; 32]>,
        },
        ReceiveKeysend {
                payment_preimage: PaymentPreimage,
+               payment_metadata: Option<Vec<u8>>,
                incoming_cltv_expiry: u32, // Used to track when we should expire pending HTLCs that go unclaimed
        },
 }
@@ -280,13 +282,12 @@ impl_writeable_tlv_based_enum!(SentHTLCId,
 pub(crate) enum HTLCSource {
        PreviousHopData(HTLCPreviousHopData),
        OutboundRoute {
-               path: Vec<RouteHop>,
+               path: Path,
                session_priv: SecretKey,
                /// Technically we can recalculate this from the route, but we cache it here to avoid
                /// doing a double-pass on route when we get a failure back
                first_hop_htlc_msat: u64,
                payment_id: PaymentId,
-               payment_secret: Option<PaymentSecret>,
        },
 }
 #[allow(clippy::derive_hash_xor_eq)] // Our Hash is faithful to the data, we just don't have SecretKey::hash
@@ -297,27 +298,37 @@ impl core::hash::Hash for HTLCSource {
                                0u8.hash(hasher);
                                prev_hop_data.hash(hasher);
                        },
-                       HTLCSource::OutboundRoute { path, session_priv, payment_id, payment_secret, first_hop_htlc_msat } => {
+                       HTLCSource::OutboundRoute { path, session_priv, payment_id, first_hop_htlc_msat } => {
                                1u8.hash(hasher);
                                path.hash(hasher);
                                session_priv[..].hash(hasher);
                                payment_id.hash(hasher);
-                               payment_secret.hash(hasher);
                                first_hop_htlc_msat.hash(hasher);
                        },
                }
        }
 }
-#[cfg(not(feature = "grind_signatures"))]
-#[cfg(test)]
 impl HTLCSource {
+       #[cfg(not(feature = "grind_signatures"))]
+       #[cfg(test)]
        pub fn dummy() -> Self {
                HTLCSource::OutboundRoute {
-                       path: Vec::new(),
+                       path: Path { hops: Vec::new(), blinded_tail: None },
                        session_priv: SecretKey::from_slice(&[1; 32]).unwrap(),
                        first_hop_htlc_msat: 0,
                        payment_id: PaymentId([2; 32]),
-                       payment_secret: None,
+               }
+       }
+
+       #[cfg(debug_assertions)]
+       /// Checks whether this HTLCSource could possibly match the given HTLC output in a commitment
+       /// transaction. Useful to ensure different datastructures match up.
+       pub(crate) fn possibly_matches_output(&self, htlc: &super::chan_utils::HTLCOutputInCommitment) -> bool {
+               if let HTLCSource::OutboundRoute { first_hop_htlc_msat, .. } = self {
+                       *first_hop_htlc_msat == htlc.amount_msat
+               } else {
+                       // There's nothing we can check for forwarded HTLCs
+                       true
                }
        }
 }
@@ -461,6 +472,12 @@ impl_writeable_tlv_based!(ClaimingPayment, {
        (4, receiver_node_id, required),
 });
 
+struct ClaimablePayment {
+       purpose: events::PaymentPurpose,
+       onion_fields: Option<RecipientOnionFields>,
+       htlcs: Vec<ClaimableHTLC>,
+}
+
 /// Information about claimable or being-claimed payments
 struct ClaimablePayments {
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
@@ -471,7 +488,7 @@ struct ClaimablePayments {
        ///
        /// When adding to the map, [`Self::pending_claiming_payments`] must also be checked to ensure
        /// we don't get a duplicate payment.
-       claimable_htlcs: HashMap<PaymentHash, (events::PaymentPurpose, Vec<ClaimableHTLC>)>,
+       claimable_payments: HashMap<PaymentHash, ClaimablePayment>,
 
        /// Map from payment hash to the payment data for HTLCs which we have begun claiming, but which
        /// are waiting on a [`ChannelMonitorUpdate`] to complete in order to be surfaced to the user
@@ -609,6 +626,61 @@ pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<
 /// This is not exported to bindings users as Arcs don't make sense in bindings
 pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, 'f, 'g, 'h, M, T, F, L> = ChannelManager<&'a M, &'b T, &'c KeysManager, &'c KeysManager, &'c KeysManager, &'d F, &'e DefaultRouter<&'f NetworkGraph<&'g L>, &'g L, &'h Mutex<ProbabilisticScorer<&'f NetworkGraph<&'g L>, &'g L>>>, &'g L>;
 
+/// A trivial trait which describes any [`ChannelManager`] used in testing.
+#[cfg(any(test, feature = "_test_utils"))]
+pub trait AChannelManager {
+       type Watch: chain::Watch<Self::Signer>;
+       type M: Deref<Target = Self::Watch>;
+       type Broadcaster: BroadcasterInterface;
+       type T: Deref<Target = Self::Broadcaster>;
+       type EntropySource: EntropySource;
+       type ES: Deref<Target = Self::EntropySource>;
+       type NodeSigner: NodeSigner;
+       type NS: Deref<Target = Self::NodeSigner>;
+       type Signer: WriteableEcdsaChannelSigner;
+       type SignerProvider: SignerProvider<Signer = Self::Signer>;
+       type SP: Deref<Target = Self::SignerProvider>;
+       type FeeEstimator: FeeEstimator;
+       type F: Deref<Target = Self::FeeEstimator>;
+       type Router: Router;
+       type R: Deref<Target = Self::Router>;
+       type Logger: Logger;
+       type L: Deref<Target = Self::Logger>;
+       fn get_cm(&self) -> &ChannelManager<Self::M, Self::T, Self::ES, Self::NS, Self::SP, Self::F, Self::R, Self::L>;
+}
+#[cfg(any(test, feature = "_test_utils"))]
+impl<M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: Deref, R: Deref, L: Deref> AChannelManager
+for ChannelManager<M, T, ES, NS, SP, F, R, L>
+where
+       M::Target: chain::Watch<<SP::Target as SignerProvider>::Signer> + Sized,
+       T::Target: BroadcasterInterface + Sized,
+       ES::Target: EntropySource + Sized,
+       NS::Target: NodeSigner + Sized,
+       SP::Target: SignerProvider + Sized,
+       F::Target: FeeEstimator + Sized,
+       R::Target: Router + Sized,
+       L::Target: Logger + Sized,
+{
+       type Watch = M::Target;
+       type M = M;
+       type Broadcaster = T::Target;
+       type T = T;
+       type EntropySource = ES::Target;
+       type ES = ES;
+       type NodeSigner = NS::Target;
+       type NS = NS;
+       type Signer = <SP::Target as SignerProvider>::Signer;
+       type SignerProvider = SP::Target;
+       type SP = SP;
+       type FeeEstimator = F::Target;
+       type F = F;
+       type Router = R::Target;
+       type R = R;
+       type Logger = L::Target;
+       type L = L;
+       fn get_cm(&self) -> &ChannelManager<M, T, ES, NS, SP, F, R, L> { self }
+}
+
 /// Manager which keeps track of a number of channels and sends messages to the appropriate
 /// channel, also tracking HTLC preimages and forwarding onion packets appropriately.
 ///
@@ -862,6 +934,8 @@ where
 
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        pending_events: Mutex<Vec<events::Event>>,
+       /// A simple atomic flag to ensure only one task at a time can be processing events asynchronously.
+       pending_events_processor: AtomicBool,
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        pending_background_events: Mutex<Vec<BackgroundEvent>>,
        /// Used when we have to take a BIG lock to make sure everything is self-consistent.
@@ -1006,6 +1080,14 @@ pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3;
 /// [`OutboundPayments::remove_stale_resolved_payments`].
 pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7;
 
+/// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is disconnected
+/// until we mark the channel disabled and gossip the update.
+pub(crate) const DISABLE_GOSSIP_TICKS: u8 = 10;
+
+/// The number of ticks of [`ChannelManager::timer_tick_occurred`] where a peer is connected until
+/// we mark the channel enabled and gossip the update.
+pub(crate) const ENABLE_GOSSIP_TICKS: u8 = 5;
+
 /// The maximum number of unfunded channels we can have per-peer before we start rejecting new
 /// (inbound) ones. The number of peers with unfunded channels is limited separately in
 /// [`MAX_UNFUNDED_CHANNEL_PEERS`].
@@ -1345,15 +1427,15 @@ pub struct PhantomRouteHints {
 }
 
 macro_rules! handle_error {
-       ($self: ident, $internal: expr, $counterparty_node_id: expr) => {
+       ($self: ident, $internal: expr, $counterparty_node_id: expr) => { {
+               // In testing, ensure there are no deadlocks where the lock is already held upon
+               // entering the macro.
+               debug_assert_ne!($self.pending_events.held_by_thread(), LockHeldState::HeldByThread);
+               debug_assert_ne!($self.per_peer_state.held_by_thread(), LockHeldState::HeldByThread);
+
                match $internal {
                        Ok(msg) => Ok(msg),
                        Err(MsgHandleErrInternal { err, chan_id, shutdown_finish }) => {
-                               // In testing, ensure there are no deadlocks where the lock is already held upon
-                               // entering the macro.
-                               debug_assert_ne!($self.pending_events.held_by_thread(), LockHeldState::HeldByThread);
-                               debug_assert_ne!($self.per_peer_state.held_by_thread(), LockHeldState::HeldByThread);
-
                                let mut msg_events = Vec::with_capacity(2);
 
                                if let Some((shutdown_res, update_option)) = shutdown_finish {
@@ -1392,7 +1474,7 @@ macro_rules! handle_error {
                                Err(err)
                        },
                }
-       }
+       } }
 }
 
 macro_rules! update_maps_on_chan_removal {
@@ -1496,18 +1578,31 @@ macro_rules! send_channel_ready {
        }}
 }
 
+macro_rules! emit_channel_pending_event {
+       ($locked_events: expr, $channel: expr) => {
+               if $channel.should_emit_channel_pending_event() {
+                       $locked_events.push(events::Event::ChannelPending {
+                               channel_id: $channel.channel_id(),
+                               former_temporary_channel_id: $channel.temporary_channel_id(),
+                               counterparty_node_id: $channel.get_counterparty_node_id(),
+                               user_channel_id: $channel.get_user_id(),
+                               funding_txo: $channel.get_funding_txo().unwrap().into_bitcoin_outpoint(),
+                       });
+                       $channel.set_channel_pending_event_emitted();
+               }
+       }
+}
+
 macro_rules! emit_channel_ready_event {
-       ($self: expr, $channel: expr) => {
+       ($locked_events: expr, $channel: expr) => {
                if $channel.should_emit_channel_ready_event() {
-                       {
-                               let mut pending_events = $self.pending_events.lock().unwrap();
-                               pending_events.push(events::Event::ChannelReady {
-                                       channel_id: $channel.channel_id(),
-                                       user_channel_id: $channel.get_user_id(),
-                                       counterparty_node_id: $channel.get_counterparty_node_id(),
-                                       channel_type: $channel.get_channel_type().clone(),
-                               });
-                       }
+                       debug_assert!($channel.channel_pending_event_emitted());
+                       $locked_events.push(events::Event::ChannelReady {
+                               channel_id: $channel.channel_id(),
+                               user_channel_id: $channel.get_user_id(),
+                               counterparty_node_id: $channel.get_counterparty_node_id(),
+                               channel_type: $channel.get_channel_type().clone(),
+                       });
                        $channel.set_channel_ready_event_emitted();
                }
        }
@@ -1566,7 +1661,7 @@ macro_rules! handle_new_monitor_update {
        ($self: ident, $update_res: expr, $update_id: expr, $peer_state_lock: expr, $peer_state: expr, $per_peer_state_lock: expr, $chan: expr, MANUALLY_REMOVING, $remove: expr) => { {
                // update_maps_on_chan_removal needs to be able to take id_to_peer, so make sure we can in
                // any case so that it won't deadlock.
-               debug_assert!($self.id_to_peer.try_lock().is_ok());
+               debug_assert_ne!($self.id_to_peer.held_by_thread(), LockHeldState::HeldByThread);
                match $update_res {
                        ChannelMonitorUpdateStatus::InProgress => {
                                log_debug!($self.logger, "ChannelMonitor update for {} in flight, holding messages until the update completes.",
@@ -1601,6 +1696,53 @@ macro_rules! handle_new_monitor_update {
        }
 }
 
+macro_rules! process_events_body {
+       ($self: expr, $event_to_handle: expr, $handle_event: expr) => {
+               let mut processed_all_events = false;
+               while !processed_all_events {
+                       if $self.pending_events_processor.compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed).is_err() {
+                               return;
+                       }
+
+                       let mut result = NotifyOption::SkipPersist;
+
+                       {
+                               // We'll acquire our total consistency lock so that we can be sure no other
+                               // persists happen while processing monitor events.
+                               let _read_guard = $self.total_consistency_lock.read().unwrap();
+
+                               // TODO: This behavior should be documented. It's unintuitive that we query
+                               // ChannelMonitors when clearing other events.
+                               if $self.process_pending_monitor_events() {
+                                       result = NotifyOption::DoPersist;
+                               }
+                       }
+
+                       let pending_events = $self.pending_events.lock().unwrap().clone();
+                       let num_events = pending_events.len();
+                       if !pending_events.is_empty() {
+                               result = NotifyOption::DoPersist;
+                       }
+
+                       for event in pending_events {
+                               $event_to_handle = event;
+                               $handle_event;
+                       }
+
+                       {
+                               let mut pending_events = $self.pending_events.lock().unwrap();
+                               pending_events.drain(..num_events);
+                               processed_all_events = pending_events.is_empty();
+                               $self.pending_events_processor.store(false, Ordering::Release);
+                       }
+
+                       if result == NotifyOption::DoPersist {
+                               $self.persistence_notifier.notify();
+                       }
+               }
+       }
+}
+
 impl<M: Deref, T: Deref, ES: Deref, NS: Deref, SP: Deref, F: Deref, R: Deref, L: Deref> ChannelManager<M, T, ES, NS, SP, F, R, L>
 where
        M::Target: chain::Watch<<SP::Target as SignerProvider>::Signer>,
@@ -1646,7 +1788,7 @@ where
                        pending_inbound_payments: Mutex::new(HashMap::new()),
                        pending_outbound_payments: OutboundPayments::new(),
                        forward_htlcs: Mutex::new(HashMap::new()),
-                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs: HashMap::new(), pending_claiming_payments: HashMap::new() }),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_payments: HashMap::new(), pending_claiming_payments: HashMap::new() }),
                        pending_intercepted_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
                        short_to_chan_info: FairRwLock::new(HashMap::new()),
@@ -1664,6 +1806,7 @@ where
                        per_peer_state: FairRwLock::new(HashMap::new()),
 
                        pending_events: Mutex::new(Vec::new()),
+                       pending_events_processor: AtomicBool::new(false),
                        pending_background_events: Mutex::new(Vec::new()),
                        total_consistency_lock: RwLock::new(()),
                        persistence_notifier: Notifier::new(),
@@ -2141,7 +2284,7 @@ where
                                        msg: "Got non final data with an HMAC of 0",
                                });
                        },
-                       msgs::OnionHopDataFormat::FinalNode { payment_data, keysend_preimage } => {
+                       msgs::OnionHopDataFormat::FinalNode { payment_data, keysend_preimage, payment_metadata } => {
                                if payment_data.is_some() && keysend_preimage.is_some() {
                                        return Err(ReceiveError {
                                                err_code: 0x4000|22,
@@ -2151,6 +2294,7 @@ where
                                } else if let Some(data) = payment_data {
                                        PendingHTLCRouting::Receive {
                                                payment_data: data,
+                                               payment_metadata,
                                                incoming_cltv_expiry: hop_data.outgoing_cltv_value,
                                                phantom_shared_secret,
                                        }
@@ -2171,6 +2315,7 @@ where
 
                                        PendingHTLCRouting::ReceiveKeysend {
                                                payment_preimage,
+                                               payment_metadata,
                                                incoming_cltv_expiry: hop_data.outgoing_cltv_value,
                                        }
                                } else {
@@ -2350,7 +2495,14 @@ where
                                                // hopefully an attacker trying to path-trace payments cannot make this occur
                                                // on a small/per-node/per-channel scale.
                                                if !chan.is_live() { // channel_disabled
-                                                       break Some(("Forwarding channel is not in a ready state.", 0x1000 | 20, chan_update_opt));
+                                                       // If the channel_update we're going to return is disabled (i.e. the
+                                                       // peer has been disabled for some time), return `channel_disabled`,
+                                                       // otherwise return `temporary_channel_failure`.
+                                                       if chan_update_opt.as_ref().map(|u| u.contents.flags & 2 == 2).unwrap_or(false) {
+                                                               break Some(("Forwarding channel has been disconnected for some time.", 0x1000 | 20, chan_update_opt));
+                                                       } else {
+                                                               break Some(("Forwarding channel is not in a ready state.", 0x1000 | 7, chan_update_opt));
+                                                       }
                                                }
                                                if *outgoing_amt_msat < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
                                                        break Some(("HTLC amount was below the htlc_minimum_msat", 0x1000 | 11, chan_update_opt));
@@ -2475,11 +2627,18 @@ where
                log_trace!(self.logger, "Generating channel update for channel {}", log_bytes!(chan.channel_id()));
                let were_node_one = self.our_network_pubkey.serialize()[..] < chan.get_counterparty_node_id().serialize()[..];
 
+               let enabled = chan.is_usable() && match chan.channel_update_status() {
+                       ChannelUpdateStatus::Enabled => true,
+                       ChannelUpdateStatus::DisabledStaged(_) => true,
+                       ChannelUpdateStatus::Disabled => false,
+                       ChannelUpdateStatus::EnabledStaged(_) => false,
+               };
+
                let unsigned = msgs::UnsignedChannelUpdate {
                        chain_hash: self.genesis_hash,
                        short_channel_id,
                        timestamp: chan.get_update_time_counter(),
-                       flags: (!were_node_one) as u8 | ((!chan.is_live() as u8) << 1),
+                       flags: (!were_node_one) as u8 | ((!enabled as u8) << 1),
                        cltv_expiry_delta: chan.get_cltv_expiry_delta(),
                        htlc_minimum_msat: chan.get_counterparty_htlc_minimum_msat(),
                        htlc_maximum_msat: chan.get_announced_htlc_max_msat(),
@@ -2500,29 +2659,29 @@ where
        }
 
        #[cfg(test)]
-       pub(crate) fn test_send_payment_along_path(&self, path: &Vec<RouteHop>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
+       pub(crate) fn test_send_payment_along_path(&self, path: &Path, payment_hash: &PaymentHash, recipient_onion: RecipientOnionFields, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
                let _lck = self.total_consistency_lock.read().unwrap();
-               self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv_bytes)
+               self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv_bytes)
        }
 
-       fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
+       fn send_payment_along_path(&self, path: &Path, payment_hash: &PaymentHash, recipient_onion: RecipientOnionFields, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
                // The top-level caller should hold the total_consistency_lock read lock.
                debug_assert!(self.total_consistency_lock.try_write().is_err());
 
-               log_trace!(self.logger, "Attempting to send payment for path with next hop {}", path.first().unwrap().short_channel_id);
+               log_trace!(self.logger, "Attempting to send payment for path with next hop {}", path.hops.first().unwrap().short_channel_id);
                let prng_seed = self.entropy_source.get_secure_random_bytes();
                let session_priv = SecretKey::from_slice(&session_priv_bytes[..]).expect("RNG is busted");
 
                let onion_keys = onion_utils::construct_onion_keys(&self.secp_ctx, &path, &session_priv)
                        .map_err(|_| APIError::InvalidRoute{err: "Pubkey along hop was maliciously selected".to_owned()})?;
-               let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(path, total_value, payment_secret, cur_height, keysend_preimage)?;
+               let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(path, total_value, recipient_onion, cur_height, keysend_preimage)?;
                if onion_utils::route_size_insane(&onion_payloads) {
                        return Err(APIError::InvalidRoute{err: "Route size too large considering onion data".to_owned()});
                }
                let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, prng_seed, payment_hash);
 
                let err: Result<(), _> = loop {
-                       let (counterparty_node_id, id) = match self.short_to_chan_info.read().unwrap().get(&path.first().unwrap().short_channel_id) {
+                       let (counterparty_node_id, id) = match self.short_to_chan_info.read().unwrap().get(&path.hops.first().unwrap().short_channel_id) {
                                None => return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()}),
                                Some((cp_id, chan_id)) => (cp_id.clone(), chan_id.clone()),
                        };
@@ -2543,7 +2702,6 @@ where
                                                session_priv: session_priv.clone(),
                                                first_hop_htlc_msat: htlc_msat,
                                                payment_id,
-                                               payment_secret: payment_secret.clone(),
                                        }, onion_packet, &self.logger);
                                match break_chan_entry!(self, send_res, chan) {
                                        Some(monitor_update) => {
@@ -2574,7 +2732,7 @@ where
                        return Ok(());
                };
 
-               match handle_error!(self, err, path.first().unwrap().pubkey) {
+               match handle_error!(self, err, path.hops.first().unwrap().pubkey) {
                        Ok(_) => unreachable!(),
                        Err(e) => {
                                Err(APIError::ChannelUnavailable { err: e.err })
@@ -2628,59 +2786,52 @@ where
        /// irrevocably committed to on our end. In such a case, do NOT retry the payment with a
        /// different route unless you intend to pay twice!
        ///
-       /// # A caution on `payment_secret`
-       ///
-       /// `payment_secret` is unrelated to `payment_hash` (or [`PaymentPreimage`]) and exists to
-       /// authenticate the sender to the recipient and prevent payment-probing (deanonymization)
-       /// attacks. For newer nodes, it will be provided to you in the invoice. If you do not have one,
-       /// the [`Route`] must not contain multiple paths as multi-path payments require a
-       /// recipient-provided `payment_secret`.
-       ///
-       /// If a `payment_secret` *is* provided, we assume that the invoice had the payment_secret
-       /// feature bit set (either as required or as available). If multiple paths are present in the
-       /// [`Route`], we assume the invoice had the basic_mpp feature set.
-       ///
        /// [`Event::PaymentSent`]: events::Event::PaymentSent
        /// [`Event::PaymentFailed`]: events::Event::PaymentFailed
        /// [`UpdateHTLCs`]: events::MessageSendEvent::UpdateHTLCs
        /// [`PeerManager::process_events`]: crate::ln::peer_handler::PeerManager::process_events
        /// [`ChannelMonitorUpdateStatus::InProgress`]: crate::chain::ChannelMonitorUpdateStatus::InProgress
-       pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
+       pub fn send_payment_with_route(&self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
                self.pending_outbound_payments
-                       .send_payment_with_route(route, payment_hash, payment_secret, payment_id, &self.entropy_source, &self.node_signer, best_block_height,
-                               |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                               self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+                       .send_payment_with_route(route, payment_hash, recipient_onion, payment_id, &self.entropy_source, &self.node_signer, best_block_height,
+                               |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                               self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Similar to [`ChannelManager::send_payment`], but will automatically find a route based on
        /// `route_params` and retry failed payment paths based on `retry_strategy`.
-       pub fn send_payment_with_retry(&self, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<(), RetryableSendFailure> {
+       pub fn send_payment(&self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<(), RetryableSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
                self.pending_outbound_payments
-                       .send_payment(payment_hash, payment_secret, payment_id, retry_strategy, route_params,
+                       .send_payment(payment_hash, recipient_onion, payment_id, retry_strategy, route_params,
                                &self.router, self.list_usable_channels(), || self.compute_inflight_htlcs(),
                                &self.entropy_source, &self.node_signer, best_block_height, &self.logger,
                                &self.pending_events,
-                               |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                               self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+                               |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                               self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        #[cfg(test)]
-       pub(super) fn test_send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
+       pub(super) fn test_send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               self.pending_outbound_payments.test_send_payment_internal(route, payment_hash, payment_secret, keysend_preimage, payment_id, recv_value_msat, onion_session_privs, &self.node_signer, best_block_height,
-                       |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                       self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+               self.pending_outbound_payments.test_send_payment_internal(route, payment_hash, recipient_onion, keysend_preimage, payment_id, recv_value_msat, onion_session_privs, &self.node_signer, best_block_height,
+                       |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        #[cfg(test)]
-       pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
+       pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
-               self.pending_outbound_payments.test_add_new_pending_payment(payment_hash, payment_secret, payment_id, route, None, &self.entropy_source, best_block_height)
+               self.pending_outbound_payments.test_add_new_pending_payment(payment_hash, recipient_onion, payment_id, route, None, &self.entropy_source, best_block_height)
+       }
+
+       #[cfg(test)]
+       pub(crate) fn test_set_payment_metadata(&self, payment_id: PaymentId, new_payment_metadata: Option<Vec<u8>>) {
+               self.pending_outbound_payments.test_set_payment_metadata(payment_id, new_payment_metadata);
        }
 
 
@@ -2702,7 +2853,7 @@ where
        /// [`Event::PaymentSent`]: events::Event::PaymentSent
        pub fn abandon_payment(&self, payment_id: PaymentId) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               self.pending_outbound_payments.abandon_payment(payment_id, &self.pending_events);
+               self.pending_outbound_payments.abandon_payment(payment_id, PaymentFailureReason::UserAbandoned, &self.pending_events);
        }
 
        /// Send a spontaneous payment, which is a payment that does not require the recipient to have
@@ -2720,14 +2871,14 @@ where
        /// Note that `route` must have exactly one path.
        ///
        /// [`send_payment`]: Self::send_payment
-       pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId) -> Result<PaymentHash, PaymentSendFailure> {
+       pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>, recipient_onion: RecipientOnionFields, payment_id: PaymentId) -> Result<PaymentHash, PaymentSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
                self.pending_outbound_payments.send_spontaneous_payment_with_route(
-                       route, payment_preimage, payment_id, &self.entropy_source, &self.node_signer,
-                       best_block_height,
-                       |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                       self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+                       route, payment_preimage, recipient_onion, payment_id, &self.entropy_source,
+                       &self.node_signer, best_block_height,
+                       |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Similar to [`ChannelManager::send_spontaneous_payment`], but will automatically find a route
@@ -2737,26 +2888,26 @@ where
        /// payments.
        ///
        /// [`PaymentParameters::for_keysend`]: crate::routing::router::PaymentParameters::for_keysend
-       pub fn send_spontaneous_payment_with_retry(&self, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<PaymentHash, RetryableSendFailure> {
+       pub fn send_spontaneous_payment_with_retry(&self, payment_preimage: Option<PaymentPreimage>, recipient_onion: RecipientOnionFields, payment_id: PaymentId, route_params: RouteParameters, retry_strategy: Retry) -> Result<PaymentHash, RetryableSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               self.pending_outbound_payments.send_spontaneous_payment(payment_preimage, payment_id,
-                       retry_strategy, route_params, &self.router, self.list_usable_channels(),
+               self.pending_outbound_payments.send_spontaneous_payment(payment_preimage, recipient_onion,
+                       payment_id, retry_strategy, route_params, &self.router, self.list_usable_channels(),
                        || self.compute_inflight_htlcs(),  &self.entropy_source, &self.node_signer, best_block_height,
                        &self.logger, &self.pending_events,
-                       |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                       self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+                       |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Send a payment that is probing the given route for liquidity. We calculate the
        /// [`PaymentHash`] of probes based on a static secret and a random [`PaymentId`], which allows
        /// us to easily discern them from real payments.
-       pub fn send_probe(&self, hops: Vec<RouteHop>) -> Result<(PaymentHash, PaymentId), PaymentSendFailure> {
+       pub fn send_probe(&self, path: Path) -> Result<(PaymentHash, PaymentId), PaymentSendFailure> {
                let best_block_height = self.best_block.read().unwrap().height();
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               self.pending_outbound_payments.send_probe(hops, self.probing_cookie_secret, &self.entropy_source, &self.node_signer, best_block_height,
-                       |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                       self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv))
+               self.pending_outbound_payments.send_probe(path, self.probing_cookie_secret, &self.entropy_source, &self.node_signer, best_block_height,
+                       |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv))
        }
 
        /// Returns whether a payment with the given [`PaymentHash`] and [`PaymentId`] is, in fact, a
@@ -2777,29 +2928,34 @@ where
 
                let mut peer_state_lock = peer_state_mutex.lock().unwrap();
                let peer_state = &mut *peer_state_lock;
-               let (chan, msg) = {
-                       let (res, chan) = {
-                               match peer_state.channel_by_id.remove(temporary_channel_id) {
-                                       Some(mut chan) => {
-                                               let funding_txo = find_funding_output(&chan, &funding_transaction)?;
-
-                                               (chan.get_outbound_funding_created(funding_transaction, funding_txo, &self.logger)
-                                                       .map_err(|e| if let ChannelError::Close(msg) = e {
-                                                               MsgHandleErrInternal::from_finish_shutdown(msg, chan.channel_id(), chan.get_user_id(), chan.force_shutdown(true), None)
-                                                       } else { unreachable!(); })
-                                               , chan)
+               let (msg, chan) = match peer_state.channel_by_id.remove(temporary_channel_id) {
+                       Some(mut chan) => {
+                               let funding_txo = find_funding_output(&chan, &funding_transaction)?;
+
+                               let funding_res = chan.get_outbound_funding_created(funding_transaction, funding_txo, &self.logger)
+                                       .map_err(|e| if let ChannelError::Close(msg) = e {
+                                               MsgHandleErrInternal::from_finish_shutdown(msg, chan.channel_id(), chan.get_user_id(), chan.force_shutdown(true), None)
+                                       } else { unreachable!(); });
+                               match funding_res {
+                                       Ok(funding_msg) => (funding_msg, chan),
+                                       Err(_) => {
+                                               mem::drop(peer_state_lock);
+                                               mem::drop(per_peer_state);
+
+                                               let _ = handle_error!(self, funding_res, chan.get_counterparty_node_id());
+                                               return Err(APIError::ChannelUnavailable {
+                                                       err: "Signer refused to sign the initial commitment transaction".to_owned()
+                                               });
                                        },
-                                       None => { return Err(APIError::ChannelUnavailable { err: format!("Channel with id {} not found for the passed counterparty node_id {}", log_bytes!(*temporary_channel_id), counterparty_node_id) }) },
                                }
-                       };
-                       match handle_error!(self, res, chan.get_counterparty_node_id()) {
-                               Ok(funding_msg) => {
-                                       (chan, funding_msg)
-                               },
-                               Err(_) => { return Err(APIError::ChannelUnavailable {
-                                       err: "Signer refused to sign the initial commitment transaction".to_owned()
-                               }) },
-                       }
+                       },
+                       None => {
+                               return Err(APIError::ChannelUnavailable {
+                                       err: format!(
+                                               "Channel with id {} not found for the passed counterparty node_id {}",
+                                               log_bytes!(*temporary_channel_id), counterparty_node_id),
+                               })
+                       },
                };
 
                peer_state.pending_msg_events.push(events::MessageSendEvent::SendFundingCreated {
@@ -2870,10 +3026,11 @@ where
                }
                {
                        let height = self.best_block.read().unwrap().height();
-                       // Transactions are evaluated as final by network mempools at the next block. However, the modules
-                       // constituting our Lightning node might not have perfect sync about their blockchain views. Thus, if
-                       // the wallet module is in advance on the LDK view, allow one more block of headroom.
-                       if !funding_transaction.input.iter().all(|input| input.sequence == Sequence::MAX) && LockTime::from(funding_transaction.lock_time).is_block_height() && funding_transaction.lock_time.0 > height + 2 {
+                       // Transactions are evaluated as final by network mempools if their locktime is strictly
+                       // lower than the next block height. However, the modules constituting our Lightning
+                       // node might not have perfect sync about their blockchain views. Thus, if the wallet
+                       // module is ahead of LDK, only allow one more block of headroom.
+                       if !funding_transaction.input.iter().all(|input| input.sequence == Sequence::MAX) && LockTime::from(funding_transaction.lock_time).is_block_height() && funding_transaction.lock_time.0 > height + 1 {
                                return Err(APIError::APIMisuseError {
                                        err: "Funding transaction absolute timelock is non-final".to_owned()
                                });
@@ -3262,7 +3419,7 @@ where
                                                }
                                        }
                                } else {
-                                       for forward_info in pending_forwards.drain(..) {
+                                       'next_forwardable_htlc: for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
                                                        HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                                                prev_short_channel_id, prev_htlc_id, prev_funding_outpoint, prev_user_channel_id,
@@ -3270,13 +3427,19 @@ where
                                                                        routing, incoming_shared_secret, payment_hash, incoming_amt_msat, outgoing_amt_msat, ..
                                                                }
                                                        }) => {
-                                                               let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
-                                                                       PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
+                                                               let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret, mut onion_fields) = match routing {
+                                                                       PendingHTLCRouting::Receive { payment_data, payment_metadata, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
-                                                                               (incoming_cltv_expiry, OnionPayload::Invoice { _legacy_hop_data }, Some(payment_data), phantom_shared_secret)
+                                                                               let onion_fields =
+                                                                                       RecipientOnionFields { payment_secret: Some(payment_data.payment_secret), payment_metadata };
+                                                                               (incoming_cltv_expiry, OnionPayload::Invoice { _legacy_hop_data },
+                                                                                       Some(payment_data), phantom_shared_secret, onion_fields)
+                                                                       },
+                                                                       PendingHTLCRouting::ReceiveKeysend { payment_preimage, payment_metadata, incoming_cltv_expiry } => {
+                                                                               let onion_fields = RecipientOnionFields { payment_secret: None, payment_metadata };
+                                                                               (incoming_cltv_expiry, OnionPayload::Spontaneous(payment_preimage),
+                                                                                       None, None, onion_fields)
                                                                        },
-                                                                       PendingHTLCRouting::ReceiveKeysend { payment_preimage, incoming_cltv_expiry } =>
-                                                                               (incoming_cltv_expiry, OnionPayload::Spontaneous(payment_preimage), None, None),
                                                                        _ => {
                                                                                panic!("short_channel_id == 0 should imply any pending_forward entries are of type Receive");
                                                                        }
@@ -3301,8 +3464,11 @@ where
                                                                        onion_payload,
                                                                };
 
+                                                               let mut committed_to_claimable = false;
+
                                                                macro_rules! fail_htlc {
                                                                        ($htlc: expr, $payment_hash: expr) => {
+                                                                               debug_assert!(!committed_to_claimable);
                                                                                let mut htlc_msat_height_data = $htlc.value.to_be_bytes().to_vec();
                                                                                htlc_msat_height_data.extend_from_slice(
                                                                                        &self.best_block.read().unwrap().height().to_be_bytes(),
@@ -3317,6 +3483,7 @@ where
                                                                                        HTLCFailReason::reason(0x4000 | 15, htlc_msat_height_data),
                                                                                        HTLCDestination::FailedPayment { payment_hash: $payment_hash },
                                                                                ));
+                                                                               continue 'next_forwardable_htlc;
                                                                        }
                                                                }
                                                                let phantom_shared_secret = claimable_htlc.prev_hop.phantom_shared_secret;
@@ -3338,20 +3505,35 @@ where
                                                                                let mut claimable_payments = self.claimable_payments.lock().unwrap();
                                                                                if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                       continue
                                                                                }
-                                                                               let (_, ref mut htlcs) = claimable_payments.claimable_htlcs.entry(payment_hash)
-                                                                                       .or_insert_with(|| (purpose(), Vec::new()));
+                                                                               let ref mut claimable_payment = claimable_payments.claimable_payments
+                                                                                       .entry(payment_hash)
+                                                                                       // Note that if we insert here we MUST NOT fail_htlc!()
+                                                                                       .or_insert_with(|| {
+                                                                                               committed_to_claimable = true;
+                                                                                               ClaimablePayment {
+                                                                                                       purpose: purpose(), htlcs: Vec::new(), onion_fields: None,
+                                                                                               }
+                                                                                       });
+                                                                               if let Some(earlier_fields) = &mut claimable_payment.onion_fields {
+                                                                                       if earlier_fields.check_merge(&mut onion_fields).is_err() {
+                                                                                               fail_htlc!(claimable_htlc, payment_hash);
+                                                                                       }
+                                                                               } else {
+                                                                                       claimable_payment.onion_fields = Some(onion_fields);
+                                                                               }
+                                                                               let ref mut htlcs = &mut claimable_payment.htlcs;
                                                                                if htlcs.len() == 1 {
                                                                                        if let OnionPayload::Spontaneous(_) = htlcs[0].onion_payload {
                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as we already had an existing keysend HTLC with the same payment hash", log_bytes!(payment_hash.0));
                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                               continue
                                                                                        }
                                                                                }
                                                                                let mut total_value = claimable_htlc.sender_intended_value;
+                                                                               let mut earliest_expiry = claimable_htlc.cltv_expiry;
                                                                                for htlc in htlcs.iter() {
                                                                                        total_value += htlc.sender_intended_value;
+                                                                                       earliest_expiry = cmp::min(earliest_expiry, htlc.cltv_expiry);
                                                                                        match &htlc.onion_payload {
                                                                                                OnionPayload::Invoice { .. } => {
                                                                                                        if htlc.total_msat != $payment_data.total_msat {
@@ -3373,6 +3555,9 @@ where
                                                                                                log_bytes!(payment_hash.0));
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
                                                                                } else if total_value >= $payment_data.total_msat {
+                                                                                       #[allow(unused_assignments)] {
+                                                                                               committed_to_claimable = true;
+                                                                                       }
                                                                                        let prev_channel_id = prev_funding_outpoint.to_channel_id();
                                                                                        htlcs.push(claimable_htlc);
                                                                                        let amount_msat = htlcs.iter().map(|htlc| htlc.value).sum();
@@ -3384,6 +3569,8 @@ where
                                                                                                amount_msat,
                                                                                                via_channel_id: Some(prev_channel_id),
                                                                                                via_user_channel_id: Some(prev_user_channel_id),
+                                                                                               claim_deadline: Some(earliest_expiry - HTLC_FAIL_BACK_BUFFER),
+                                                                                               onion_fields: claimable_payment.onion_fields.clone(),
                                                                                        });
                                                                                        payment_claimable_generated = true;
                                                                                } else {
@@ -3391,6 +3578,9 @@ where
                                                                                        // payment value yet, wait until we receive more
                                                                                        // MPP parts.
                                                                                        htlcs.push(claimable_htlc);
+                                                                                       #[allow(unused_assignments)] {
+                                                                                               committed_to_claimable = true;
+                                                                                       }
                                                                                }
                                                                                payment_claimable_generated
                                                                        }}
@@ -3413,7 +3603,6 @@ where
                                                                                                        Err(()) => {
                                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as payment verification failed", log_bytes!(payment_hash.0));
                                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                               continue
                                                                                                        }
                                                                                                };
                                                                                                if let Some(min_final_cltv_expiry_delta) = min_final_cltv_expiry_delta {
@@ -3422,7 +3611,6 @@ where
                                                                                                                log_trace!(self.logger, "Failing new HTLC with payment_hash {} as its CLTV expiry was too soon (had {}, earliest expected {})",
                                                                                                                        log_bytes!(payment_hash.0), cltv_expiry, expected_min_expiry_height);
                                                                                                                fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                               continue;
                                                                                                        }
                                                                                                }
                                                                                                check_total_value!(payment_data, payment_preimage);
@@ -3431,14 +3619,18 @@ where
                                                                                                let mut claimable_payments = self.claimable_payments.lock().unwrap();
                                                                                                if claimable_payments.pending_claiming_payments.contains_key(&payment_hash) {
                                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                                       continue
                                                                                                }
-                                                                                               match claimable_payments.claimable_htlcs.entry(payment_hash) {
+                                                                                               match claimable_payments.claimable_payments.entry(payment_hash) {
                                                                                                        hash_map::Entry::Vacant(e) => {
                                                                                                                let amount_msat = claimable_htlc.value;
                                                                                                                claimable_htlc.total_value_received = Some(amount_msat);
+                                                                                                               let claim_deadline = Some(claimable_htlc.cltv_expiry - HTLC_FAIL_BACK_BUFFER);
                                                                                                                let purpose = events::PaymentPurpose::SpontaneousPayment(preimage);
-                                                                                                               e.insert((purpose.clone(), vec![claimable_htlc]));
+                                                                                                               e.insert(ClaimablePayment {
+                                                                                                                       purpose: purpose.clone(),
+                                                                                                                       onion_fields: Some(onion_fields.clone()),
+                                                                                                                       htlcs: vec![claimable_htlc],
+                                                                                                               });
                                                                                                                let prev_channel_id = prev_funding_outpoint.to_channel_id();
                                                                                                                new_events.push(events::Event::PaymentClaimable {
                                                                                                                        receiver_node_id: Some(receiver_node_id),
@@ -3447,6 +3639,8 @@ where
                                                                                                                        purpose,
                                                                                                                        via_channel_id: Some(prev_channel_id),
                                                                                                                        via_user_channel_id: Some(prev_user_channel_id),
+                                                                                                                       claim_deadline,
+                                                                                                                       onion_fields: Some(onion_fields),
                                                                                                                });
                                                                                                        },
                                                                                                        hash_map::Entry::Occupied(_) => {
@@ -3461,7 +3655,6 @@ where
                                                                                if payment_data.is_none() {
                                                                                        log_trace!(self.logger, "Failing new keysend HTLC with payment_hash {} because we already have an inbound payment with the same payment hash", log_bytes!(payment_hash.0));
                                                                                        fail_htlc!(claimable_htlc, payment_hash);
-                                                                                       continue
                                                                                };
                                                                                let payment_data = payment_data.unwrap();
                                                                                if inbound_payment.get().payment_secret != payment_data.payment_secret {
@@ -3493,8 +3686,8 @@ where
                self.pending_outbound_payments.check_retry_payments(&self.router, || self.list_usable_channels(),
                        || self.compute_inflight_htlcs(), &self.entropy_source, &self.node_signer, best_block_height,
                        &self.pending_events, &self.logger,
-                       |path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv|
-                       self.send_payment_along_path(path, payment_hash, payment_secret, total_value, cur_height, payment_id, keysend_preimage, session_priv));
+                       |path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv|
+                       self.send_payment_along_path(path, payment_hash, recipient_onion, total_value, cur_height, payment_id, keysend_preimage, session_priv));
 
                for (htlc_source, payment_hash, failure_reason, destination) in failed_forwards.drain(..) {
                        self.fail_htlc_backwards_internal(&htlc_source, &payment_hash, &failure_reason, destination);
@@ -3632,27 +3825,39 @@ where
                                                }
 
                                                match chan.channel_update_status() {
-                                                       ChannelUpdateStatus::Enabled if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged),
-                                                       ChannelUpdateStatus::Disabled if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged),
-                                                       ChannelUpdateStatus::DisabledStaged if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::Enabled),
-                                                       ChannelUpdateStatus::EnabledStaged if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::Disabled),
-                                                       ChannelUpdateStatus::DisabledStaged if !chan.is_live() => {
-                                                               if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
-                                                                       pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                                               msg: update
-                                                                       });
+                                                       ChannelUpdateStatus::Enabled if !chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged(0)),
+                                                       ChannelUpdateStatus::Disabled if chan.is_live() => chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged(0)),
+                                                       ChannelUpdateStatus::DisabledStaged(_) if chan.is_live()
+                                                               => chan.set_channel_update_status(ChannelUpdateStatus::Enabled),
+                                                       ChannelUpdateStatus::EnabledStaged(_) if !chan.is_live()
+                                                               => chan.set_channel_update_status(ChannelUpdateStatus::Disabled),
+                                                       ChannelUpdateStatus::DisabledStaged(mut n) if !chan.is_live() => {
+                                                               n += 1;
+                                                               if n >= DISABLE_GOSSIP_TICKS {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::Disabled);
+                                                                       if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
+                                                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
+                                                                                       msg: update
+                                                                               });
+                                                                       }
+                                                                       should_persist = NotifyOption::DoPersist;
+                                                               } else {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::DisabledStaged(n));
                                                                }
-                                                               should_persist = NotifyOption::DoPersist;
-                                                               chan.set_channel_update_status(ChannelUpdateStatus::Disabled);
                                                        },
-                                                       ChannelUpdateStatus::EnabledStaged if chan.is_live() => {
-                                                               if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
-                                                                       pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
-                                                                               msg: update
-                                                                       });
+                                                       ChannelUpdateStatus::EnabledStaged(mut n) if chan.is_live() => {
+                                                               n += 1;
+                                                               if n >= ENABLE_GOSSIP_TICKS {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::Enabled);
+                                                                       if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
+                                                                               pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
+                                                                                       msg: update
+                                                                               });
+                                                                       }
+                                                                       should_persist = NotifyOption::DoPersist;
+                                                               } else {
+                                                                       chan.set_channel_update_status(ChannelUpdateStatus::EnabledStaged(n));
                                                                }
-                                                               should_persist = NotifyOption::DoPersist;
-                                                               chan.set_channel_update_status(ChannelUpdateStatus::Enabled);
                                                        },
                                                        _ => {},
                                                }
@@ -3694,24 +3899,27 @@ where
                                }
                        }
 
-                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                               if htlcs.is_empty() {
+                       self.claimable_payments.lock().unwrap().claimable_payments.retain(|payment_hash, payment| {
+                               if payment.htlcs.is_empty() {
                                        // This should be unreachable
                                        debug_assert!(false);
                                        return false;
                                }
-                               if let OnionPayload::Invoice { .. } = htlcs[0].onion_payload {
+                               if let OnionPayload::Invoice { .. } = payment.htlcs[0].onion_payload {
                                        // Check if we've received all the parts we need for an MPP (the value of the parts adds to total_msat).
                                        // In this case we're not going to handle any timeouts of the parts here.
                                        // This condition determining whether the MPP is complete here must match
                                        // exactly the condition used in `process_pending_htlc_forwards`.
-                                       if htlcs[0].total_msat <= htlcs.iter().fold(0, |total, htlc| total + htlc.sender_intended_value) {
+                                       if payment.htlcs[0].total_msat <= payment.htlcs.iter()
+                                               .fold(0, |total, htlc| total + htlc.sender_intended_value)
+                                       {
                                                return true;
-                                       } else if htlcs.into_iter().any(|htlc| {
+                                       } else if payment.htlcs.iter_mut().any(|htlc| {
                                                htlc.timer_ticks += 1;
                                                return htlc.timer_ticks >= MPP_TIMEOUT_TICKS
                                        }) {
-                                               timed_out_mpp_htlcs.extend(htlcs.drain(..).map(|htlc: ClaimableHTLC| (htlc.prev_hop, *payment_hash)));
+                                               timed_out_mpp_htlcs.extend(payment.htlcs.drain(..)
+                                                       .map(|htlc: ClaimableHTLC| (htlc.prev_hop, *payment_hash)));
                                                return false;
                                        }
                                }
@@ -3766,9 +3974,9 @@ where
        pub fn fail_htlc_backwards_with_reason(&self, payment_hash: &PaymentHash, failure_code: FailureCode) {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
-               let removed_source = self.claimable_payments.lock().unwrap().claimable_htlcs.remove(payment_hash);
-               if let Some((_, mut sources)) = removed_source {
-                       for htlc in sources.drain(..) {
+               let removed_source = self.claimable_payments.lock().unwrap().claimable_payments.remove(payment_hash);
+               if let Some(payment) = removed_source {
+                       for htlc in payment.htlcs {
                                let reason = self.get_htlc_fail_reason_from_failure_code(failure_code, &htlc);
                                let source = HTLCSource::PreviousHopData(htlc.prev_hop);
                                let receiver = HTLCDestination::FailedPayment { payment_hash: *payment_hash };
@@ -3922,9 +4130,10 @@ where
        /// Provides a payment preimage in response to [`Event::PaymentClaimable`], generating any
        /// [`MessageSendEvent`]s needed to claim the payment.
        ///
-       /// Note that calling this method does *not* guarantee that the payment has been claimed. You
-       /// *must* wait for an [`Event::PaymentClaimed`] event which upon a successful claim will be
-       /// provided to your [`EventHandler`] when [`process_pending_events`] is next called.
+       /// This method is guaranteed to ensure the payment has been claimed but only if the current
+       /// height is strictly below [`Event::PaymentClaimable::claim_deadline`]. To avoid race
+       /// conditions, you should wait for an [`Event::PaymentClaimed`] before considering the payment
+       /// successful. It will generally be available in the next [`process_pending_events`] call.
        ///
        /// Note that if you did not set an `amount_msat` when calling [`create_inbound_payment`] or
        /// [`create_inbound_payment_for_hash`] you must check that the amount in the `PaymentClaimable`
@@ -3932,6 +4141,7 @@ where
        /// the sender "proof-of-payment" when they did not fulfill the full expected payment.
        ///
        /// [`Event::PaymentClaimable`]: crate::events::Event::PaymentClaimable
+       /// [`Event::PaymentClaimable::claim_deadline`]: crate::events::Event::PaymentClaimable::claim_deadline
        /// [`Event::PaymentClaimed`]: crate::events::Event::PaymentClaimed
        /// [`process_pending_events`]: EventsProvider::process_pending_events
        /// [`create_inbound_payment`]: Self::create_inbound_payment
@@ -3943,9 +4153,9 @@ where
 
                let mut sources = {
                        let mut claimable_payments = self.claimable_payments.lock().unwrap();
-                       if let Some((payment_purpose, sources)) = claimable_payments.claimable_htlcs.remove(&payment_hash) {
+                       if let Some(payment) = claimable_payments.claimable_payments.remove(&payment_hash) {
                                let mut receiver_node_id = self.our_network_pubkey;
-                               for htlc in sources.iter() {
+                               for htlc in payment.htlcs.iter() {
                                        if htlc.prev_hop.phantom_shared_secret.is_some() {
                                                let phantom_pubkey = self.node_signer.get_node_id(Recipient::PhantomNode)
                                                        .expect("Failed to get node_id for phantom node recipient");
@@ -3955,34 +4165,23 @@ where
                                }
 
                                let dup_purpose = claimable_payments.pending_claiming_payments.insert(payment_hash,
-                                       ClaimingPayment { amount_msat: sources.iter().map(|source| source.value).sum(),
-                                       payment_purpose, receiver_node_id,
+                                       ClaimingPayment { amount_msat: payment.htlcs.iter().map(|source| source.value).sum(),
+                                       payment_purpose: payment.purpose, receiver_node_id,
                                });
                                if dup_purpose.is_some() {
                                        debug_assert!(false, "Shouldn't get a duplicate pending claim event ever");
                                        log_error!(self.logger, "Got a duplicate pending claimable event on payment hash {}! Please report this bug",
                                                log_bytes!(payment_hash.0));
                                }
-                               sources
+                               payment.htlcs
                        } else { return; }
                };
                debug_assert!(!sources.is_empty());
 
-               // If we are claiming an MPP payment, we check that all channels which contain a claimable
-               // HTLC still exist. While this isn't guaranteed to remain true if a channel closes while
-               // we're claiming (or even after we claim, before the commitment update dance completes),
-               // it should be a relatively rare race, and we'd rather not claim HTLCs that require us to
-               // go on-chain (and lose the on-chain fee to do so) than just reject the payment.
-               //
-               // Note that we'll still always get our funds - as long as the generated
-               // `ChannelMonitorUpdate` makes it out to the relevant monitor we can claim on-chain.
-               //
-               // If we find an HTLC which we would need to claim but for which we do not have a
-               // channel, we will fail all parts of the MPP payment. While we could wait and see if
-               // the sender retries the already-failed path(s), it should be a pretty rare case where
-               // we got all the HTLCs and then a channel closed while we were waiting for the user to
-               // provide the preimage, so worrying too much about the optimal handling isn't worth
-               // it.
+               // Just in case one HTLC has been failed between when we generated the `PaymentClaimable`
+               // and when we got here we need to check that the amount we're about to claim matches the
+               // amount we told the user in the last `PaymentClaimable`. We also do a sanity-check that
+               // the MPP parts all have the same `total_msat`.
                let mut claimable_amt_msat = 0;
                let mut prev_total_msat = None;
                let mut expected_amt_msat = None;
@@ -3990,28 +4189,6 @@ where
                let mut errs = Vec::new();
                let per_peer_state = self.per_peer_state.read().unwrap();
                for htlc in sources.iter() {
-                       let (counterparty_node_id, chan_id) = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
-                               Some((cp_id, chan_id)) => (cp_id.clone(), chan_id.clone()),
-                               None => {
-                                       valid_mpp = false;
-                                       break;
-                               }
-                       };
-
-                       let peer_state_mutex_opt = per_peer_state.get(&counterparty_node_id);
-                       if peer_state_mutex_opt.is_none() {
-                               valid_mpp = false;
-                               break;
-                       }
-
-                       let mut peer_state_lock = peer_state_mutex_opt.unwrap().lock().unwrap();
-                       let peer_state = &mut *peer_state_lock;
-
-                       if peer_state.channel_by_id.get(&chan_id).is_none() {
-                               valid_mpp = false;
-                               break;
-                       }
-
                        if prev_total_msat.is_some() && prev_total_msat != Some(htlc.total_msat) {
                                log_error!(self.logger, "Somehow ended up with an MPP payment with different expected total amounts - this should not be reachable!");
                                debug_assert!(false);
@@ -4091,45 +4268,46 @@ where
        -> Result<(), (PublicKey, MsgHandleErrInternal)> {
                //TODO: Delay the claimed_funds relaying just like we do outbound relay!
 
-               let per_peer_state = self.per_peer_state.read().unwrap();
-               let chan_id = prev_hop.outpoint.to_channel_id();
-               let counterparty_node_id_opt = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
-                       Some((cp_id, _dup_chan_id)) => Some(cp_id.clone()),
-                       None => None
-               };
+               {
+                       let per_peer_state = self.per_peer_state.read().unwrap();
+                       let chan_id = prev_hop.outpoint.to_channel_id();
+                       let counterparty_node_id_opt = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
+                               Some((cp_id, _dup_chan_id)) => Some(cp_id.clone()),
+                               None => None
+                       };
 
-               let peer_state_opt = counterparty_node_id_opt.as_ref().map(
-                       |counterparty_node_id| per_peer_state.get(counterparty_node_id).map(
-                               |peer_mutex| peer_mutex.lock().unwrap()
-                       )
-               ).unwrap_or(None);
+                       let peer_state_opt = counterparty_node_id_opt.as_ref().map(
+                               |counterparty_node_id| per_peer_state.get(counterparty_node_id)
+                                       .map(|peer_mutex| peer_mutex.lock().unwrap())
+                       ).unwrap_or(None);
 
-               if peer_state_opt.is_some() {
-                       let mut peer_state_lock = peer_state_opt.unwrap();
-                       let peer_state = &mut *peer_state_lock;
-                       if let hash_map::Entry::Occupied(mut chan) = peer_state.channel_by_id.entry(chan_id) {
-                               let counterparty_node_id = chan.get().get_counterparty_node_id();
-                               let fulfill_res = chan.get_mut().get_update_fulfill_htlc_and_commit(prev_hop.htlc_id, payment_preimage, &self.logger);
-
-                               if let UpdateFulfillCommitFetch::NewClaim { htlc_value_msat, monitor_update } = fulfill_res {
-                                       if let Some(action) = completion_action(Some(htlc_value_msat)) {
-                                               log_trace!(self.logger, "Tracking monitor update completion action for channel {}: {:?}",
-                                                       log_bytes!(chan_id), action);
-                                               peer_state.monitor_update_blocked_actions.entry(chan_id).or_insert(Vec::new()).push(action);
-                                       }
-                                       let update_id = monitor_update.update_id;
-                                       let update_res = self.chain_monitor.update_channel(prev_hop.outpoint, monitor_update);
-                                       let res = handle_new_monitor_update!(self, update_res, update_id, peer_state_lock,
-                                               peer_state, per_peer_state, chan);
-                                       if let Err(e) = res {
-                                               // TODO: This is a *critical* error - we probably updated the outbound edge
-                                               // of the HTLC's monitor with a preimage. We should retry this monitor
-                                               // update over and over again until morale improves.
-                                               log_error!(self.logger, "Failed to update channel monitor with preimage {:?}", payment_preimage);
-                                               return Err((counterparty_node_id, e));
+                       if peer_state_opt.is_some() {
+                               let mut peer_state_lock = peer_state_opt.unwrap();
+                               let peer_state = &mut *peer_state_lock;
+                               if let hash_map::Entry::Occupied(mut chan) = peer_state.channel_by_id.entry(chan_id) {
+                                       let counterparty_node_id = chan.get().get_counterparty_node_id();
+                                       let fulfill_res = chan.get_mut().get_update_fulfill_htlc_and_commit(prev_hop.htlc_id, payment_preimage, &self.logger);
+
+                                       if let UpdateFulfillCommitFetch::NewClaim { htlc_value_msat, monitor_update } = fulfill_res {
+                                               if let Some(action) = completion_action(Some(htlc_value_msat)) {
+                                                       log_trace!(self.logger, "Tracking monitor update completion action for channel {}: {:?}",
+                                                               log_bytes!(chan_id), action);
+                                                       peer_state.monitor_update_blocked_actions.entry(chan_id).or_insert(Vec::new()).push(action);
+                                               }
+                                               let update_id = monitor_update.update_id;
+                                               let update_res = self.chain_monitor.update_channel(prev_hop.outpoint, monitor_update);
+                                               let res = handle_new_monitor_update!(self, update_res, update_id, peer_state_lock,
+                                                       peer_state, per_peer_state, chan);
+                                               if let Err(e) = res {
+                                                       // TODO: This is a *critical* error - we probably updated the outbound edge
+                                                       // of the HTLC's monitor with a preimage. We should retry this monitor
+                                                       // update over and over again until morale improves.
+                                                       log_error!(self.logger, "Failed to update channel monitor with preimage {:?}", payment_preimage);
+                                                       return Err((counterparty_node_id, e));
+                                               }
                                        }
+                                       return Ok(());
                                }
-                               return Ok(());
                        }
                }
                let preimage_update = ChannelMonitorUpdate {
@@ -4253,8 +4431,6 @@ where
                        });
                }
 
-               emit_channel_ready_event!(self, channel);
-
                macro_rules! handle_cs { () => {
                        if let Some(update) = commitment_update {
                                pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
@@ -4287,6 +4463,12 @@ where
                        self.tx_broadcaster.broadcast_transaction(&tx);
                }
 
+               {
+                       let mut pending_events = self.pending_events.lock().unwrap();
+                       emit_channel_pending_event!(pending_events, channel);
+                       emit_channel_ready_event!(pending_events, channel);
+               }
+
                htlc_forwards
        }
 
@@ -4711,7 +4893,10 @@ where
                                        }
                                }
 
-                               emit_channel_ready_event!(self, chan.get_mut());
+                               {
+                                       let mut pending_events = self.pending_events.lock().unwrap();
+                                       emit_channel_ready_event!(pending_events, chan.get_mut());
+                               }
 
                                Ok(())
                        },
@@ -5724,30 +5909,8 @@ where
        pub async fn process_pending_events_async<Future: core::future::Future, H: Fn(Event) -> Future>(
                &self, handler: H
        ) {
-               // We'll acquire our total consistency lock until the returned future completes so that
-               // we can be sure no other persists happen while processing events.
-               let _read_guard = self.total_consistency_lock.read().unwrap();
-
-               let mut result = NotifyOption::SkipPersist;
-
-               // TODO: This behavior should be documented. It's unintuitive that we query
-               // ChannelMonitors when clearing other events.
-               if self.process_pending_monitor_events() {
-                       result = NotifyOption::DoPersist;
-               }
-
-               let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
-               if !pending_events.is_empty() {
-                       result = NotifyOption::DoPersist;
-               }
-
-               for event in pending_events {
-                       handler(event).await;
-               }
-
-               if result == NotifyOption::DoPersist {
-                       self.persistence_notifier.notify();
-               }
+               let mut ev;
+               process_events_body!(self, ev, { handler(ev).await });
        }
 }
 
@@ -5829,26 +5992,8 @@ where
        /// An [`EventHandler`] may safely call back to the provider in order to handle an event.
        /// However, it must not call [`Writeable::write`] as doing so would result in a deadlock.
        fn process_pending_events<H: Deref>(&self, handler: H) where H::Target: EventHandler {
-               PersistenceNotifierGuard::optionally_notify(&self.total_consistency_lock, &self.persistence_notifier, || {
-                       let mut result = NotifyOption::SkipPersist;
-
-                       // TODO: This behavior should be documented. It's unintuitive that we query
-                       // ChannelMonitors when clearing other events.
-                       if self.process_pending_monitor_events() {
-                               result = NotifyOption::DoPersist;
-                       }
-
-                       let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
-                       if !pending_events.is_empty() {
-                               result = NotifyOption::DoPersist;
-                       }
-
-                       for event in pending_events {
-                               handler.handle_event(event);
-                       }
-
-                       result
-               });
+               let mut ev;
+               process_events_body!(self, ev, handler.handle_event(ev));
        }
 }
 
@@ -6036,7 +6181,10 @@ where
                                                        }
                                                }
 
-                                               emit_channel_ready_event!(self, channel);
+                                               {
+                                                       let mut pending_events = self.pending_events.lock().unwrap();
+                                                       emit_channel_ready_event!(pending_events, channel);
+                                               }
 
                                                if let Some(announcement_sigs) = announcement_sigs {
                                                        log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(channel.channel_id()));
@@ -6097,8 +6245,8 @@ where
                }
 
                if let Some(height) = height_opt {
-                       self.claimable_payments.lock().unwrap().claimable_htlcs.retain(|payment_hash, (_, htlcs)| {
-                               htlcs.retain(|htlc| {
+                       self.claimable_payments.lock().unwrap().claimable_payments.retain(|payment_hash, payment| {
+                               payment.htlcs.retain(|htlc| {
                                        // If height is approaching the number of blocks we think it takes us to get
                                        // our commitment transaction confirmed before the HTLC expires, plus the
                                        // number of blocks we generally consider it to take to do a commitment update,
@@ -6113,7 +6261,7 @@ where
                                                false
                                        } else { true }
                                });
-                               !htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
+                               !payment.htlcs.is_empty() // Only retain this entry if htlcs has at least one entry.
                        });
 
                        let mut intercepted_htlcs = self.pending_intercepted_htlcs.lock().unwrap();
@@ -6147,34 +6295,11 @@ where
                }
        }
 
-       /// Blocks until ChannelManager needs to be persisted or a timeout is reached. It returns a bool
-       /// indicating whether persistence is necessary. Only one listener on
-       /// [`await_persistable_update`], [`await_persistable_update_timeout`], or a future returned by
-       /// [`get_persistable_update_future`] is guaranteed to be woken up.
+       /// Gets a [`Future`] that completes when this [`ChannelManager`] needs to be persisted.
        ///
-       /// Note that this method is not available with the `no-std` feature.
+       /// Note that callbacks registered on the [`Future`] MUST NOT call back into this
+       /// [`ChannelManager`] and should instead register actions to be taken later.
        ///
-       /// [`await_persistable_update`]: Self::await_persistable_update
-       /// [`await_persistable_update_timeout`]: Self::await_persistable_update_timeout
-       /// [`get_persistable_update_future`]: Self::get_persistable_update_future
-       #[cfg(any(test, feature = "std"))]
-       pub fn await_persistable_update_timeout(&self, max_wait: Duration) -> bool {
-               self.persistence_notifier.wait_timeout(max_wait)
-       }
-
-       /// Blocks until ChannelManager needs to be persisted. Only one listener on
-       /// [`await_persistable_update`], `await_persistable_update_timeout`, or a future returned by
-       /// [`get_persistable_update_future`] is guaranteed to be woken up.
-       ///
-       /// [`await_persistable_update`]: Self::await_persistable_update
-       /// [`get_persistable_update_future`]: Self::get_persistable_update_future
-       pub fn await_persistable_update(&self) {
-               self.persistence_notifier.wait()
-       }
-
-       /// Gets a [`Future`] that completes when a persistable update is available. Note that
-       /// callbacks registered on the [`Future`] MUST NOT call back into this [`ChannelManager`] and
-       /// should instead register actions to be taken later.
        pub fn get_persistable_update_future(&self) -> Future {
                self.persistence_notifier.get_future()
        }
@@ -6721,10 +6846,12 @@ impl_writeable_tlv_based_enum!(PendingHTLCRouting,
                (0, payment_data, required),
                (1, phantom_shared_secret, option),
                (2, incoming_cltv_expiry, required),
+               (3, payment_metadata, option),
        },
        (2, ReceiveKeysend) => {
                (0, payment_preimage, required),
                (2, incoming_cltv_expiry, required),
+               (3, payment_metadata, option),
        },
 ;);
 
@@ -6898,30 +7025,30 @@ impl Readable for HTLCSource {
                        0 => {
                                let mut session_priv: crate::util::ser::RequiredWrapper<SecretKey> = crate::util::ser::RequiredWrapper(None);
                                let mut first_hop_htlc_msat: u64 = 0;
-                               let mut path: Option<Vec<RouteHop>> = Some(Vec::new());
+                               let mut path_hops: Option<Vec<RouteHop>> = Some(Vec::new());
                                let mut payment_id = None;
-                               let mut payment_secret = None;
                                let mut payment_params: Option<PaymentParameters> = None;
+                               let mut blinded_tail: Option<BlindedTail> = None;
                                read_tlv_fields!(reader, {
                                        (0, session_priv, required),
                                        (1, payment_id, option),
                                        (2, first_hop_htlc_msat, required),
-                                       (3, payment_secret, option),
-                                       (4, path, vec_type),
+                                       (4, path_hops, vec_type),
                                        (5, payment_params, (option: ReadableArgs, 0)),
+                                       (6, blinded_tail, option),
                                });
                                if payment_id.is_none() {
                                        // For backwards compat, if there was no payment_id written, use the session_priv bytes
                                        // instead.
                                        payment_id = Some(PaymentId(*session_priv.0.unwrap().as_ref()));
                                }
-                               if path.is_none() || path.as_ref().unwrap().is_empty() {
+                               let path = Path { hops: path_hops.ok_or(DecodeError::InvalidValue)?, blinded_tail };
+                               if path.hops.len() == 0 {
                                        return Err(DecodeError::InvalidValue);
                                }
-                               let path = path.unwrap();
                                if let Some(params) = payment_params.as_mut() {
                                        if params.final_cltv_expiry_delta == 0 {
-                                               params.final_cltv_expiry_delta = path.last().unwrap().cltv_expiry_delta;
+                                               params.final_cltv_expiry_delta = path.final_cltv_expiry_delta().ok_or(DecodeError::InvalidValue)?;
                                        }
                                }
                                Ok(HTLCSource::OutboundRoute {
@@ -6929,7 +7056,6 @@ impl Readable for HTLCSource {
                                        first_hop_htlc_msat,
                                        path,
                                        payment_id: payment_id.unwrap(),
-                                       payment_secret,
                                })
                        }
                        1 => Ok(HTLCSource::PreviousHopData(Readable::read(reader)?)),
@@ -6941,16 +7067,17 @@ impl Readable for HTLCSource {
 impl Writeable for HTLCSource {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), crate::io::Error> {
                match self {
-                       HTLCSource::OutboundRoute { ref session_priv, ref first_hop_htlc_msat, ref path, payment_id, payment_secret } => {
+                       HTLCSource::OutboundRoute { ref session_priv, ref first_hop_htlc_msat, ref path, payment_id } => {
                                0u8.write(writer)?;
                                let payment_id_opt = Some(payment_id);
                                write_tlv_fields!(writer, {
                                        (0, session_priv, required),
                                        (1, payment_id_opt, option),
                                        (2, first_hop_htlc_msat, required),
-                                       (3, payment_secret, option),
-                                       (4, *path, vec_type),
+                                       // 3 was previously used to write a PaymentSecret for the payment.
+                                       (4, path.hops, vec_type),
                                        (5, None::<PaymentParameters>, option), // payment_params in LDK versions prior to 0.0.115
+                                       (6, path.blinded_tail, option),
                                 });
                        }
                        HTLCSource::PreviousHopData(ref field) => {
@@ -7060,14 +7187,16 @@ where
                let pending_outbound_payments = self.pending_outbound_payments.pending_outbound_payments.lock().unwrap();
 
                let mut htlc_purposes: Vec<&events::PaymentPurpose> = Vec::new();
-               (claimable_payments.claimable_htlcs.len() as u64).write(writer)?;
-               for (payment_hash, (purpose, previous_hops)) in claimable_payments.claimable_htlcs.iter() {
+               let mut htlc_onion_fields: Vec<&_> = Vec::new();
+               (claimable_payments.claimable_payments.len() as u64).write(writer)?;
+               for (payment_hash, payment) in claimable_payments.claimable_payments.iter() {
                        payment_hash.write(writer)?;
-                       (previous_hops.len() as u64).write(writer)?;
-                       for htlc in previous_hops.iter() {
+                       (payment.htlcs.len() as u64).write(writer)?;
+                       for htlc in payment.htlcs.iter() {
                                htlc.write(writer)?;
                        }
-                       htlc_purposes.push(purpose);
+                       htlc_purposes.push(&payment.purpose);
+                       htlc_onion_fields.push(&payment.onion_fields);
                }
 
                let mut monitor_update_blocked_actions_per_peer = None;
@@ -7182,6 +7311,7 @@ where
                        (7, self.fake_scid_rand_bytes, required),
                        (9, htlc_purposes, vec_type),
                        (11, self.probing_cookie_secret, required),
+                       (13, htlc_onion_fields, optional_vec),
                });
 
                Ok(())
@@ -7560,6 +7690,7 @@ where
                let mut fake_scid_rand_bytes: Option<[u8; 32]> = None;
                let mut probing_cookie_secret: Option<[u8; 32]> = None;
                let mut claimable_htlc_purposes = None;
+               let mut claimable_htlc_onion_fields = None;
                let mut pending_claiming_payments = Some(HashMap::new());
                let mut monitor_update_blocked_actions_per_peer = Some(Vec::new());
                read_tlv_fields!(reader, {
@@ -7572,6 +7703,7 @@ where
                        (7, fake_scid_rand_bytes, option),
                        (9, claimable_htlc_purposes, vec_type),
                        (11, probing_cookie_secret, option),
+                       (13, claimable_htlc_onion_fields, optional_vec),
                });
                if fake_scid_rand_bytes.is_none() {
                        fake_scid_rand_bytes = Some(args.entropy_source.get_secure_random_bytes());
@@ -7611,13 +7743,13 @@ where
                        for (_, monitor) in args.channel_monitors.iter() {
                                if id_to_peer.get(&monitor.get_funding_txo().0.to_channel_id()).is_none() {
                                        for (htlc_source, (htlc, _)) in monitor.get_pending_or_resolved_outbound_htlcs() {
-                                               if let HTLCSource::OutboundRoute { payment_id, session_priv, path, payment_secret, .. } = htlc_source {
-                                                       if path.is_empty() {
+                                               if let HTLCSource::OutboundRoute { payment_id, session_priv, path, .. } = htlc_source {
+                                                       if path.hops.is_empty() {
                                                                log_error!(args.logger, "Got an empty path for a pending payment");
                                                                return Err(DecodeError::InvalidValue);
                                                        }
 
-                                                       let path_amt = path.last().unwrap().fee_msat;
+                                                       let path_amt = path.final_value_msat();
                                                        let mut session_priv_bytes = [0; 32];
                                                        session_priv_bytes[..].copy_from_slice(&session_priv[..]);
                                                        match pending_outbounds.pending_outbound_payments.lock().unwrap().entry(payment_id) {
@@ -7627,14 +7759,15 @@ where
                                                                                if newly_added { "Added" } else { "Had" }, path_amt, log_bytes!(session_priv_bytes), log_bytes!(htlc.payment_hash.0));
                                                                },
                                                                hash_map::Entry::Vacant(entry) => {
-                                                                       let path_fee = path.get_path_fees();
+                                                                       let path_fee = path.fee_msat();
                                                                        entry.insert(PendingOutboundPayment::Retryable {
                                                                                retry_strategy: None,
                                                                                attempts: PaymentAttempts::new(),
                                                                                payment_params: None,
                                                                                session_privs: [session_priv_bytes].iter().map(|a| *a).collect(),
                                                                                payment_hash: htlc.payment_hash,
-                                                                               payment_secret,
+                                                                               payment_secret: None, // only used for retries, and we'll never retry on startup
+                                                                               payment_metadata: None, // only used for retries, and we'll never retry on startup
                                                                                keysend_preimage: None, // only used for retries, and we'll never retry on startup
                                                                                pending_amt_msat: path_amt,
                                                                                pending_fee_msat: Some(path_fee),
@@ -7719,22 +7852,39 @@ where
                let inbound_pmt_key_material = args.node_signer.get_inbound_payment_key_material();
                let expanded_inbound_key = inbound_payment::ExpandedKey::new(&inbound_pmt_key_material);
 
-               let mut claimable_htlcs = HashMap::with_capacity(claimable_htlcs_list.len());
-               if let Some(mut purposes) = claimable_htlc_purposes {
+               let mut claimable_payments = HashMap::with_capacity(claimable_htlcs_list.len());
+               if let Some(purposes) = claimable_htlc_purposes {
                        if purposes.len() != claimable_htlcs_list.len() {
                                return Err(DecodeError::InvalidValue);
                        }
-                       for (purpose, (payment_hash, previous_hops)) in purposes.drain(..).zip(claimable_htlcs_list.drain(..)) {
-                               claimable_htlcs.insert(payment_hash, (purpose, previous_hops));
+                       if let Some(onion_fields) = claimable_htlc_onion_fields {
+                               if onion_fields.len() != claimable_htlcs_list.len() {
+                                       return Err(DecodeError::InvalidValue);
+                               }
+                               for (purpose, (onion, (payment_hash, htlcs))) in
+                                       purposes.into_iter().zip(onion_fields.into_iter().zip(claimable_htlcs_list.into_iter()))
+                               {
+                                       let existing_payment = claimable_payments.insert(payment_hash, ClaimablePayment {
+                                               purpose, htlcs, onion_fields: onion,
+                                       });
+                                       if existing_payment.is_some() { return Err(DecodeError::InvalidValue); }
+                               }
+                       } else {
+                               for (purpose, (payment_hash, htlcs)) in purposes.into_iter().zip(claimable_htlcs_list.into_iter()) {
+                                       let existing_payment = claimable_payments.insert(payment_hash, ClaimablePayment {
+                                               purpose, htlcs, onion_fields: None,
+                                       });
+                                       if existing_payment.is_some() { return Err(DecodeError::InvalidValue); }
+                               }
                        }
                } else {
                        // LDK versions prior to 0.0.107 did not write a `pending_htlc_purposes`, but do
                        // include a `_legacy_hop_data` in the `OnionPayload`.
-                       for (payment_hash, previous_hops) in claimable_htlcs_list.drain(..) {
-                               if previous_hops.is_empty() {
+                       for (payment_hash, htlcs) in claimable_htlcs_list.drain(..) {
+                               if htlcs.is_empty() {
                                        return Err(DecodeError::InvalidValue);
                                }
-                               let purpose = match &previous_hops[0].onion_payload {
+                               let purpose = match &htlcs[0].onion_payload {
                                        OnionPayload::Invoice { _legacy_hop_data } => {
                                                if let Some(hop_data) = _legacy_hop_data {
                                                        events::PaymentPurpose::InvoicePayment {
@@ -7755,7 +7905,9 @@ where
                                        OnionPayload::Spontaneous(payment_preimage) =>
                                                events::PaymentPurpose::SpontaneousPayment(*payment_preimage),
                                };
-                               claimable_htlcs.insert(payment_hash, (purpose, previous_hops));
+                               claimable_payments.insert(payment_hash, ClaimablePayment {
+                                       purpose, htlcs, onion_fields: None,
+                               });
                        }
                }
 
@@ -7807,17 +7959,17 @@ where
 
                for (_, monitor) in args.channel_monitors.iter() {
                        for (payment_hash, payment_preimage) in monitor.get_stored_preimages() {
-                               if let Some((payment_purpose, claimable_htlcs)) = claimable_htlcs.remove(&payment_hash) {
+                               if let Some(payment) = claimable_payments.remove(&payment_hash) {
                                        log_info!(args.logger, "Re-claiming HTLCs with payment hash {} as we've released the preimage to a ChannelMonitor!", log_bytes!(payment_hash.0));
                                        let mut claimable_amt_msat = 0;
                                        let mut receiver_node_id = Some(our_network_pubkey);
-                                       let phantom_shared_secret = claimable_htlcs[0].prev_hop.phantom_shared_secret;
+                                       let phantom_shared_secret = payment.htlcs[0].prev_hop.phantom_shared_secret;
                                        if phantom_shared_secret.is_some() {
                                                let phantom_pubkey = args.node_signer.get_node_id(Recipient::PhantomNode)
                                                        .expect("Failed to get node_id for phantom node recipient");
                                                receiver_node_id = Some(phantom_pubkey)
                                        }
-                                       for claimable_htlc in claimable_htlcs {
+                                       for claimable_htlc in payment.htlcs {
                                                claimable_amt_msat += claimable_htlc.value;
 
                                                // Add a holding-cell claim of the payment to the Channel, which should be
@@ -7851,7 +8003,7 @@ where
                                        pending_events_read.push(events::Event::PaymentClaimed {
                                                receiver_node_id,
                                                payment_hash,
-                                               purpose: payment_purpose,
+                                               purpose: payment.purpose,
                                                amount_msat: claimable_amt_msat,
                                        });
                                }
@@ -7882,7 +8034,7 @@ where
                        pending_intercepted_htlcs: Mutex::new(pending_intercepted_htlcs.unwrap()),
 
                        forward_htlcs: Mutex::new(forward_htlcs),
-                       claimable_payments: Mutex::new(ClaimablePayments { claimable_htlcs, pending_claiming_payments: pending_claiming_payments.unwrap() }),
+                       claimable_payments: Mutex::new(ClaimablePayments { claimable_payments, pending_claiming_payments: pending_claiming_payments.unwrap() }),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
                        short_to_chan_info: FairRwLock::new(short_to_chan_info),
@@ -7898,6 +8050,7 @@ where
                        per_peer_state: FairRwLock::new(per_peer_state),
 
                        pending_events: Mutex::new(pending_events_read),
+                       pending_events_processor: AtomicBool::new(false),
                        pending_background_events: Mutex::new(pending_background_events),
                        total_consistency_lock: RwLock::new(()),
                        persistence_notifier: Notifier::new(),
@@ -7929,11 +8082,10 @@ mod tests {
        use bitcoin::hashes::Hash;
        use bitcoin::hashes::sha256::Hash as Sha256;
        use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
-       use core::time::Duration;
        use core::sync::atomic::Ordering;
        use crate::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
        use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-       use crate::ln::channelmanager::{inbound_payment, PaymentId, PaymentSendFailure, InterceptId};
+       use crate::ln::channelmanager::{inbound_payment, PaymentId, PaymentSendFailure, RecipientOnionFields, InterceptId};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs;
        use crate::ln::msgs::ChannelMessageHandler;
@@ -7954,9 +8106,9 @@ mod tests {
 
                // All nodes start with a persistable update pending as `create_network` connects each node
                // with all other nodes to make most tests simpler.
-               assert!(nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(nodes[2].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(nodes[1].node.get_persistable_update_future().poll_is_complete());
+               assert!(nodes[2].node.get_persistable_update_future().poll_is_complete());
 
                let mut chan = create_announced_chan_between_nodes(&nodes, 0, 1);
 
@@ -7970,19 +8122,19 @@ mod tests {
                        &nodes[0].node.get_our_node_id()).pop().unwrap();
 
                // The first two nodes (which opened a channel) should now require fresh persistence
-               assert!(nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(nodes[1].node.get_persistable_update_future().poll_is_complete());
                // ... but the last node should not.
-               assert!(!nodes[2].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(!nodes[2].node.get_persistable_update_future().poll_is_complete());
                // After persisting the first two nodes they should no longer need fresh persistence.
-               assert!(!nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(!nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(!nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(!nodes[1].node.get_persistable_update_future().poll_is_complete());
 
                // Node 3, unrelated to the only channel, shouldn't care if it receives a channel_update
                // about the channel.
                nodes[2].node.handle_channel_update(&nodes[1].node.get_our_node_id(), &chan.0);
                nodes[2].node.handle_channel_update(&nodes[1].node.get_our_node_id(), &chan.1);
-               assert!(!nodes[2].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(!nodes[2].node.get_persistable_update_future().poll_is_complete());
 
                // The nodes which are a party to the channel should also ignore messages from unrelated
                // parties.
@@ -7990,8 +8142,8 @@ mod tests {
                nodes[0].node.handle_channel_update(&nodes[2].node.get_our_node_id(), &chan.1);
                nodes[1].node.handle_channel_update(&nodes[2].node.get_our_node_id(), &chan.0);
                nodes[1].node.handle_channel_update(&nodes[2].node.get_our_node_id(), &chan.1);
-               assert!(!nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(!nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(!nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(!nodes[1].node.get_persistable_update_future().poll_is_complete());
 
                // At this point the channel info given by peers should still be the same.
                assert_eq!(nodes[0].node.list_channels()[0], node_a_chan_info);
@@ -8008,8 +8160,8 @@ mod tests {
                // persisted and that its channel info remains the same.
                nodes[0].node.handle_channel_update(&nodes[1].node.get_our_node_id(), &as_update);
                nodes[1].node.handle_channel_update(&nodes[0].node.get_our_node_id(), &bs_update);
-               assert!(!nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(!nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(!nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(!nodes[1].node.get_persistable_update_future().poll_is_complete());
                assert_eq!(nodes[0].node.list_channels()[0], node_a_chan_info);
                assert_eq!(nodes[1].node.list_channels()[0], node_b_chan_info);
 
@@ -8017,8 +8169,8 @@ mod tests {
                // the channel info has updated.
                nodes[0].node.handle_channel_update(&nodes[1].node.get_our_node_id(), &bs_update);
                nodes[1].node.handle_channel_update(&nodes[0].node.get_our_node_id(), &as_update);
-               assert!(nodes[0].node.await_persistable_update_timeout(Duration::from_millis(1)));
-               assert!(nodes[1].node.await_persistable_update_timeout(Duration::from_millis(1)));
+               assert!(nodes[0].node.get_persistable_update_future().poll_is_complete());
+               assert!(nodes[1].node.get_persistable_update_future().poll_is_complete());
                assert_ne!(nodes[0].node.list_channels()[0], node_a_chan_info);
                assert_ne!(nodes[1].node.list_channels()[0], node_b_chan_info);
        }
@@ -8042,15 +8194,18 @@ mod tests {
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
-               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &mpp_route).unwrap();
-               nodes[0].node.test_send_payment_along_path(&mpp_route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), payment_id, &mpp_route).unwrap();
+               nodes[0].node.test_send_payment_along_path(&mpp_route.paths[0], &our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                pass_along_path(&nodes[0], &[&nodes[1]], 200_000, our_payment_hash, Some(payment_secret), events.drain(..).next().unwrap(), false, None);
 
                // Next, send a keysend payment with the same payment_hash and make sure it fails.
-               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
+               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -8073,7 +8228,8 @@ mod tests {
                expect_payment_failed!(nodes[0], our_payment_hash, true);
 
                // Send the second half of the original MPP payment.
-               nodes[0].node.test_send_payment_along_path(&mpp_route.paths[1], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
+               nodes[0].node.test_send_payment_along_path(&mpp_route.paths[1], &our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -8170,7 +8326,8 @@ mod tests {
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
-               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
+               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -8203,7 +8360,8 @@ mod tests {
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
-               let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
+               let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage),
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -8213,7 +8371,8 @@ mod tests {
 
                // Next, attempt a regular payment and make sure it fails.
                let payment_secret = PaymentSecret([43; 32]);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -8267,8 +8426,10 @@ mod tests {
 
                let test_preimage = PaymentPreimage([42; 32]);
                let mismatch_payment_hash = PaymentHash([43; 32]);
-               let session_privs = nodes[0].node.test_add_new_pending_payment(mismatch_payment_hash, None, PaymentId(mismatch_payment_hash.0), &route).unwrap();
-               nodes[0].node.test_send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(mismatch_payment_hash,
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(mismatch_payment_hash.0), &route).unwrap();
+               nodes[0].node.test_send_payment_internal(&route, mismatch_payment_hash,
+                       RecipientOnionFields::spontaneous_empty(), Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -8310,8 +8471,11 @@ mod tests {
                let test_preimage = PaymentPreimage([42; 32]);
                let test_secret = PaymentSecret([43; 32]);
                let payment_hash = PaymentHash(Sha256::hash(&test_preimage.0).into_inner());
-               let session_privs = nodes[0].node.test_add_new_pending_payment(payment_hash, Some(test_secret), PaymentId(payment_hash.0), &route).unwrap();
-               nodes[0].node.test_send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), PaymentId(payment_hash.0), None, session_privs).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(payment_hash,
+                       RecipientOnionFields::secret_only(test_secret), PaymentId(payment_hash.0), &route).unwrap();
+               nodes[0].node.test_send_payment_internal(&route, payment_hash,
+                       RecipientOnionFields::secret_only(test_secret), Some(test_preimage),
+                       PaymentId(payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -8341,14 +8505,16 @@ mod tests {
                let (mut route, payment_hash, _, _) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
                let path = route.paths[0].clone();
                route.paths.push(path);
-               route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-               route.paths[0][0].short_channel_id = chan_1_id;
-               route.paths[0][1].short_channel_id = chan_3_id;
-               route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-               route.paths[1][0].short_channel_id = chan_2_id;
-               route.paths[1][1].short_channel_id = chan_4_id;
-
-               match nodes[0].node.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)).unwrap_err() {
+               route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+               route.paths[0].hops[0].short_channel_id = chan_1_id;
+               route.paths[0].hops[1].short_channel_id = chan_3_id;
+               route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+               route.paths[1].hops[0].short_channel_id = chan_2_id;
+               route.paths[1].hops[1].short_channel_id = chan_4_id;
+
+               match nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0))
+               .unwrap_err() {
                        PaymentSendFailure::ParameterError(APIError::APIMisuseError { ref err }) => {
                                assert!(regex::Regex::new(r"Payment secret is required for multi-path payments").unwrap().is_match(err))
                        },
@@ -8463,6 +8629,7 @@ mod tests {
                        assert_eq!(nodes_0_lock.len(), 1);
                        assert!(nodes_0_lock.contains_key(channel_id));
                }
+               expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
                {
                        // Assert that `nodes[1]`'s `id_to_peer` map is populated with the channel as soon as
@@ -8475,6 +8642,7 @@ mod tests {
                let funding_signed = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
                nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed);
                check_added_monitors!(nodes[0], 1);
+               expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
                let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
                let (announcement, nodes_0_update, nodes_1_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
                update_nodes_with_chan_announce(&nodes, 0, 1, &announcement, &nodes_0_update, &nodes_1_update);
@@ -8617,10 +8785,13 @@ mod tests {
 
                                nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
                                check_added_monitors!(nodes[1], 1);
+                               expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
+
                                let funding_signed = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
 
                                nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed);
                                check_added_monitors!(nodes[0], 1);
+                               expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
                        }
                        open_channel_msg.temporary_channel_id = nodes[0].keys_manager.get_secure_random_bytes();
                }
@@ -8838,13 +9009,13 @@ mod tests {
 pub mod bench {
        use crate::chain::Listen;
        use crate::chain::chainmonitor::{ChainMonitor, Persist};
-       use crate::chain::keysinterface::{EntropySource, KeysManager, InMemorySigner};
+       use crate::chain::keysinterface::{KeysManager, InMemorySigner};
        use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider};
-       use crate::ln::channelmanager::{self, BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage, PaymentId};
+       use crate::ln::channelmanager::{BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage, PaymentId, RecipientOnionFields, Retry};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs::{ChannelMessageHandler, Init};
        use crate::routing::gossip::NetworkGraph;
-       use crate::routing::router::{PaymentParameters, get_route};
+       use crate::routing::router::{PaymentParameters, RouteParameters};
        use crate::util::test_utils;
        use crate::util::config::UserConfig;
 
@@ -8856,14 +9027,23 @@ pub mod bench {
 
        use test::Bencher;
 
-       struct NodeHolder<'a, P: Persist<InMemorySigner>> {
-               node: &'a ChannelManager<
-                       &'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
-                               &'a test_utils::TestBroadcaster, &'a test_utils::TestFeeEstimator,
-                               &'a test_utils::TestLogger, &'a P>,
-                       &'a test_utils::TestBroadcaster, &'a KeysManager, &'a KeysManager, &'a KeysManager,
-                       &'a test_utils::TestFeeEstimator, &'a test_utils::TestRouter<'a>,
-                       &'a test_utils::TestLogger>,
+       type Manager<'a, P> = ChannelManager<
+               &'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
+                       &'a test_utils::TestBroadcaster, &'a test_utils::TestFeeEstimator,
+                       &'a test_utils::TestLogger, &'a P>,
+               &'a test_utils::TestBroadcaster, &'a KeysManager, &'a KeysManager, &'a KeysManager,
+               &'a test_utils::TestFeeEstimator, &'a test_utils::TestRouter<'a>,
+               &'a test_utils::TestLogger>;
+
+       struct ANodeHolder<'a, P: Persist<InMemorySigner>> {
+               node: &'a Manager<'a, P>,
+       }
+       impl<'a, P: Persist<InMemorySigner>> NodeHolder for ANodeHolder<'a, P> {
+               type CM = Manager<'a, P>;
+               #[inline]
+               fn node(&self) -> &Manager<'a, P> { self.node }
+               #[inline]
+               fn chain_monitor(&self) -> Option<&test_utils::TestChainMonitor> { None }
        }
 
        #[cfg(test)]
@@ -8878,7 +9058,7 @@ pub mod bench {
                // calls per node.
                let network = bitcoin::Network::Testnet;
 
-               let tx_broadcaster = test_utils::TestBroadcaster{txn_broadcasted: Mutex::new(Vec::new()), blocks: Arc::new(Mutex::new(Vec::new()))};
+               let tx_broadcaster = test_utils::TestBroadcaster::new(network);
                let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
                let logger_a = test_utils::TestLogger::with_id("node a".to_owned());
                let scorer = Mutex::new(test_utils::TestScorer::new());
@@ -8894,7 +9074,7 @@ pub mod bench {
                        network,
                        best_block: BestBlock::from_network(network),
                });
-               let node_a_holder = NodeHolder { node: &node_a };
+               let node_a_holder = ANodeHolder { node: &node_a };
 
                let logger_b = test_utils::TestLogger::with_id("node a".to_owned());
                let chain_monitor_b = ChainMonitor::new(None, &tx_broadcaster, &logger_a, &fee_estimator, &persister_b);
@@ -8904,7 +9084,7 @@ pub mod bench {
                        network,
                        best_block: BestBlock::from_network(network),
                });
-               let node_b_holder = NodeHolder { node: &node_b };
+               let node_b_holder = ANodeHolder { node: &node_b };
 
                node_a.peer_connected(&node_b.get_our_node_id(), &Init { features: node_b.init_features(), remote_network_address: None }, true).unwrap();
                node_b.peer_connected(&node_a.get_our_node_id(), &Init { features: node_a.init_features(), remote_network_address: None }, false).unwrap();
@@ -8921,7 +9101,24 @@ pub mod bench {
                } else { panic!(); }
 
                node_b.handle_funding_created(&node_a.get_our_node_id(), &get_event_msg!(node_a_holder, MessageSendEvent::SendFundingCreated, node_b.get_our_node_id()));
+               let events_b = node_b.get_and_clear_pending_events();
+               assert_eq!(events_b.len(), 1);
+               match events_b[0] {
+                       Event::ChannelPending{ ref counterparty_node_id, .. } => {
+                               assert_eq!(*counterparty_node_id, node_a.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+
                node_a.handle_funding_signed(&node_b.get_our_node_id(), &get_event_msg!(node_b_holder, MessageSendEvent::SendFundingSigned, node_a.get_our_node_id()));
+               let events_a = node_a.get_and_clear_pending_events();
+               assert_eq!(events_a.len(), 1);
+               match events_a[0] {
+                       Event::ChannelPending{ ref counterparty_node_id, .. } => {
+                               assert_eq!(*counterparty_node_id, node_b.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
 
                assert_eq!(&tx_broadcaster.txn_broadcasted.lock().unwrap()[..], &[tx.clone()]);
 
@@ -8965,40 +9162,33 @@ pub mod bench {
                        _ => panic!("Unexpected event"),
                }
 
-               let dummy_graph = NetworkGraph::new(network, &logger_a);
-
                let mut payment_count: u64 = 0;
                macro_rules! send_payment {
                        ($node_a: expr, $node_b: expr) => {
-                               let usable_channels = $node_a.list_usable_channels();
                                let payment_params = PaymentParameters::from_node_id($node_b.get_our_node_id(), TEST_FINAL_CLTV)
                                        .with_features($node_b.invoice_features());
-                               let scorer = test_utils::TestScorer::new();
-                               let seed = [3u8; 32];
-                               let keys_manager = KeysManager::new(&seed, 42, 42);
-                               let random_seed_bytes = keys_manager.get_secure_random_bytes();
-                               let route = get_route(&$node_a.get_our_node_id(), &payment_params, &dummy_graph.read_only(),
-                                       Some(&usable_channels.iter().map(|r| r).collect::<Vec<_>>()), 10_000, TEST_FINAL_CLTV, &logger_a, &scorer, &random_seed_bytes).unwrap();
-
                                let mut payment_preimage = PaymentPreimage([0; 32]);
                                payment_preimage.0[0..8].copy_from_slice(&payment_count.to_le_bytes());
                                payment_count += 1;
                                let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
                                let payment_secret = $node_b.create_inbound_payment_for_hash(payment_hash, None, 7200, None).unwrap();
 
-                               $node_a.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+                               $node_a.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                                       PaymentId(payment_hash.0), RouteParameters {
+                                               payment_params, final_value_msat: 10_000,
+                                       }, Retry::Attempts(0)).unwrap();
                                let payment_event = SendEvent::from_event($node_a.get_and_clear_pending_msg_events().pop().unwrap());
                                $node_b.handle_update_add_htlc(&$node_a.get_our_node_id(), &payment_event.msgs[0]);
                                $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &payment_event.commitment_msg);
-                               let (raa, cs) = do_get_revoke_commit_msgs!(NodeHolder { node: &$node_b }, &$node_a.get_our_node_id());
+                               let (raa, cs) = get_revoke_commit_msgs(&ANodeHolder { node: &$node_b }, &$node_a.get_our_node_id());
                                $node_a.handle_revoke_and_ack(&$node_b.get_our_node_id(), &raa);
                                $node_a.handle_commitment_signed(&$node_b.get_our_node_id(), &cs);
-                               $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &get_event_msg!(NodeHolder { node: &$node_a }, MessageSendEvent::SendRevokeAndACK, $node_b.get_our_node_id()));
+                               $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &get_event_msg!(ANodeHolder { node: &$node_a }, MessageSendEvent::SendRevokeAndACK, $node_b.get_our_node_id()));
 
-                               expect_pending_htlcs_forwardable!(NodeHolder { node: &$node_b });
-                               expect_payment_claimable!(NodeHolder { node: &$node_b }, payment_hash, payment_secret, 10_000);
+                               expect_pending_htlcs_forwardable!(ANodeHolder { node: &$node_b });
+                               expect_payment_claimable!(ANodeHolder { node: &$node_b }, payment_hash, payment_secret, 10_000);
                                $node_b.claim_funds(payment_preimage);
-                               expect_payment_claimed!(NodeHolder { node: &$node_b }, payment_hash, 10_000);
+                               expect_payment_claimed!(ANodeHolder { node: &$node_b }, payment_hash, 10_000);
 
                                match $node_b.get_and_clear_pending_msg_events().pop().unwrap() {
                                        MessageSendEvent::UpdateHTLCs { node_id, updates } => {
@@ -9009,12 +9199,12 @@ pub mod bench {
                                        _ => panic!("Failed to generate claim event"),
                                }
 
-                               let (raa, cs) = do_get_revoke_commit_msgs!(NodeHolder { node: &$node_a }, &$node_b.get_our_node_id());
+                               let (raa, cs) = get_revoke_commit_msgs(&ANodeHolder { node: &$node_a }, &$node_b.get_our_node_id());
                                $node_b.handle_revoke_and_ack(&$node_a.get_our_node_id(), &raa);
                                $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &cs);
-                               $node_a.handle_revoke_and_ack(&$node_b.get_our_node_id(), &get_event_msg!(NodeHolder { node: &$node_b }, MessageSendEvent::SendRevokeAndACK, $node_a.get_our_node_id()));
+                               $node_a.handle_revoke_and_ack(&$node_b.get_our_node_id(), &get_event_msg!(ANodeHolder { node: &$node_b }, MessageSendEvent::SendRevokeAndACK, $node_a.get_our_node_id()));
 
-                               expect_payment_sent!(NodeHolder { node: &$node_a }, payment_preimage);
+                               expect_payment_sent!(ANodeHolder { node: &$node_a }, payment_preimage);
                        }
                }
 
index b4a5c5ae52d5282135b6e664819c2b5962ea0b2c..cf375603b37b5290aaf10250218f4b66cce0678a 100644 (file)
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
 //! - `OnionMessages` - requires/supports forwarding onion messages
 //!     (see [BOLT-7](https://github.com/lightning/bolts/pull/759/files) for more information).
-//!     TODO: update link
+//     TODO: update link
 //! - `ChannelType` - node supports the channel_type field in open/accept
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
 //! - `SCIDPrivacy` - supply channel aliases for routing
 //!     (see [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md) for more information).
+//! - `PaymentMetadata` - include additional data in invoices which is passed to recipients in the
+//!      onion.
+//!      (see [BOLT-11](https://github.com/lightning/bolts/blob/master/11-payment-encoding.md) for
+//!      more).
+//! - `ZeroConf` - supports accepting HTLCs and using channels prior to funding confirmation
+//!      (see
+//!      [BOLT-2](https://github.com/lightning/bolts/blob/master/02-peer-protocol.md#the-channel_ready-message)
+//!      for more info).
 //! - `Keysend` - send funds to a node without an invoice
 //!     (see the [`Keysend` feature assignment proposal](https://github.com/lightning/bolts/issues/605#issuecomment-606679798) for more information).
 //! - `AnchorsZeroFeeHtlcTx` - requires/supports that commitment transactions include anchor outputs
-//!   and HTLC transactions are pre-signed with zero fee (see
-//!   [BOLT-3](https://github.com/lightning/bolts/blob/master/03-transactions.md) for more
-//!   information).
+//!     and HTLC transactions are pre-signed with zero fee (see
+//!     [BOLT-3](https://github.com/lightning/bolts/blob/master/03-transactions.md) for more
+//!     information).
 //!
 //! [BOLT #9]: https://github.com/lightning/bolts/blob/master/09-features.md
 //! [messages]: crate::ln::msgs
@@ -160,6 +168,14 @@ mod sealed {
                VariableLengthOnion | PaymentSecret,
                // Byte 2
                BasicMPP,
+               // Byte 3
+               ,
+               // Byte 4
+               ,
+               // Byte 5
+               ,
+               // Byte 6
+               PaymentMetadata,
        ]);
        define_context!(OfferContext, []);
        define_context!(InvoiceRequestContext, []);
@@ -259,6 +275,7 @@ mod sealed {
                                        }
 
                                        flags[Self::BYTE_OFFSET] |= Self::REQUIRED_MASK;
+                                       flags[Self::BYTE_OFFSET] &= !Self::OPTIONAL_MASK;
                                }
 
                                /// Sets the feature's optional (odd) bit in the given flags.
@@ -376,12 +393,16 @@ mod sealed {
        define_feature!(47, SCIDPrivacy, [InitContext, NodeContext, ChannelTypeContext],
                "Feature flags for only forwarding with SCID aliasing. Called `option_scid_alias` in the BOLTs",
                set_scid_privacy_optional, set_scid_privacy_required, supports_scid_privacy, requires_scid_privacy);
+       define_feature!(49, PaymentMetadata, [InvoiceContext],
+               "Feature flags for payment metadata in invoices.", set_payment_metadata_optional,
+               set_payment_metadata_required, supports_payment_metadata, requires_payment_metadata);
        define_feature!(51, ZeroConf, [InitContext, NodeContext, ChannelTypeContext],
                "Feature flags for accepting channels with zero confirmations. Called `option_zeroconf` in the BOLTs",
                set_zero_conf_optional, set_zero_conf_required, supports_zero_conf, requires_zero_conf);
        define_feature!(55, Keysend, [NodeContext],
                "Feature flags for keysend payments.", set_keysend_optional, set_keysend_required,
                supports_keysend, requires_keysend);
+       // Note: update the module-level docs when a new feature bit is added!
 
        #[cfg(test)]
        define_feature!(123456789, UnknownFeature,
@@ -885,13 +906,13 @@ mod tests {
        #[test]
        fn convert_to_context_with_unknown_flags() {
                // Ensure the `from` context has fewer known feature bytes than the `to` context.
-               assert!(<sealed::InvoiceContext as sealed::Context>::KNOWN_FEATURE_MASK.len() <
-                       <sealed::NodeContext as sealed::Context>::KNOWN_FEATURE_MASK.len());
-               let mut invoice_features = InvoiceFeatures::empty();
-               invoice_features.set_unknown_feature_optional();
-               assert!(invoice_features.supports_unknown_bits());
-               let node_features: NodeFeatures = invoice_features.to_context();
-               assert!(!node_features.supports_unknown_bits());
+               assert!(<sealed::ChannelContext as sealed::Context>::KNOWN_FEATURE_MASK.len() <
+                       <sealed::InvoiceContext as sealed::Context>::KNOWN_FEATURE_MASK.len());
+               let mut channel_features = ChannelFeatures::empty();
+               channel_features.set_unknown_feature_optional();
+               assert!(channel_features.supports_unknown_bits());
+               let invoice_features: InvoiceFeatures = channel_features.to_context_internal();
+               assert!(!invoice_features.supports_unknown_bits());
        }
 
        #[test]
index 61b927c536fd1603957b9a5d51842e5abe6b7e84..50aed2c5b4355283f77695708235c25f5d36d761 100644 (file)
@@ -13,9 +13,9 @@
 use crate::chain::{BestBlock, ChannelMonitorUpdateStatus, Confirm, Listen, Watch, keysinterface::EntropySource};
 use crate::chain::channelmonitor::ChannelMonitor;
 use crate::chain::transaction::OutPoint;
-use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose};
+use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose, PaymentFailureReason};
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-use crate::ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, PaymentId, MIN_CLTV_EXPIRY_DELTA};
+use crate::ln::channelmanager::{AChannelManager, ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, PaymentId, MIN_CLTV_EXPIRY_DELTA};
 use crate::routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate};
 use crate::routing::router::{self, PaymentParameters, Route};
 use crate::ln::features::InitFeatures;
@@ -30,7 +30,6 @@ use crate::util::config::UserConfig;
 use crate::util::ser::{ReadableArgs, Writeable};
 
 use bitcoin::blockdata::block::{Block, BlockHeader};
-use bitcoin::blockdata::constants::genesis_block;
 use bitcoin::blockdata::transaction::{Transaction, TxOut};
 use bitcoin::network::constants::Network;
 
@@ -137,6 +136,34 @@ pub enum ConnectStyle {
 }
 
 impl ConnectStyle {
+       pub fn skips_blocks(&self) -> bool {
+               match self {
+                       ConnectStyle::BestBlockFirst => false,
+                       ConnectStyle::BestBlockFirstSkippingBlocks => true,
+                       ConnectStyle::BestBlockFirstReorgsOnlyTip => true,
+                       ConnectStyle::TransactionsFirst => false,
+                       ConnectStyle::TransactionsFirstSkippingBlocks => true,
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => true,
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks => true,
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
+                       ConnectStyle::FullBlockViaListen => false,
+               }
+       }
+
+       pub fn updates_best_block_first(&self) -> bool {
+               match self {
+                       ConnectStyle::BestBlockFirst => true,
+                       ConnectStyle::BestBlockFirstSkippingBlocks => true,
+                       ConnectStyle::BestBlockFirstReorgsOnlyTip => true,
+                       ConnectStyle::TransactionsFirst => false,
+                       ConnectStyle::TransactionsFirstSkippingBlocks => false,
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => false,
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks => false,
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => false,
+                       ConnectStyle::FullBlockViaListen => false,
+               }
+       }
+
        fn random_style() -> ConnectStyle {
                #[cfg(feature = "std")] {
                        use core::hash::{BuildHasher, Hasher};
@@ -164,12 +191,7 @@ impl ConnectStyle {
 }
 
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
-       let skip_intermediaries = match *node.connect_style.borrow() {
-               ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
-                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
-                       ConnectStyle::BestBlockFirstReorgsOnlyTip|ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
-               _ => false,
-       };
+       let skip_intermediaries = node.connect_style.borrow().skips_blocks();
 
        let height = node.best_block_info().1 + 1;
        let mut block = Block {
@@ -207,6 +229,9 @@ fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: Block, sk
        #[cfg(feature = "std")] {
                eprintln!("Connecting block using Block Connection Style: {:?}", *node.connect_style.borrow());
        }
+       // Update the block internally before handing it over to LDK, to ensure our assertions regarding
+       // transaction broadcast are correct.
+       node.blocks.lock().unwrap().push((block.clone(), height));
        if !skip_intermediaries {
                let txdata: Vec<_> = block.txdata.iter().enumerate().collect();
                match *node.connect_style.borrow() {
@@ -256,7 +281,6 @@ fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: Block, sk
        }
        call_claimable_balances(node);
        node.node.test_process_background_events();
-       node.blocks.lock().unwrap().push((block, height));
 }
 
 pub fn disconnect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, count: u32) {
@@ -324,6 +348,8 @@ pub struct NodeCfg<'a> {
        pub override_init_features: Rc<RefCell<Option<InitFeatures>>>,
 }
 
+type TestChannelManager<'a, 'b, 'c> = ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'b test_utils::TestKeysInterface, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'b test_utils::TestRouter<'c>, &'c test_utils::TestLogger>;
+
 pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub chain_source: &'c test_utils::TestChainSource,
        pub tx_broadcaster: &'c test_utils::TestBroadcaster,
@@ -331,7 +357,7 @@ pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub router: &'b test_utils::TestRouter<'c>,
        pub chain_monitor: &'b test_utils::TestChainMonitor<'c>,
        pub keys_manager: &'b test_utils::TestKeysInterface,
-       pub node: &'a ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'b test_utils::TestKeysInterface, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'b test_utils::TestRouter<'c>, &'c test_utils::TestLogger>,
+       pub node: &'a TestChannelManager<'a, 'b, 'c>,
        pub network_graph: &'a NetworkGraph<&'c test_utils::TestLogger>,
        pub gossip_sync: P2PGossipSync<&'b NetworkGraph<&'c test_utils::TestLogger>, &'c test_utils::TestChainSource, &'c test_utils::TestLogger>,
        pub node_seed: [u8; 32],
@@ -367,6 +393,39 @@ impl NodePtr {
 unsafe impl Send for NodePtr {}
 unsafe impl Sync for NodePtr {}
 
+
+pub trait NodeHolder {
+       type CM: AChannelManager;
+       fn node(&self) -> &ChannelManager<
+               <Self::CM as AChannelManager>::M,
+               <Self::CM as AChannelManager>::T,
+               <Self::CM as AChannelManager>::ES,
+               <Self::CM as AChannelManager>::NS,
+               <Self::CM as AChannelManager>::SP,
+               <Self::CM as AChannelManager>::F,
+               <Self::CM as AChannelManager>::R,
+               <Self::CM as AChannelManager>::L>;
+       fn chain_monitor(&self) -> Option<&test_utils::TestChainMonitor>;
+}
+impl<H: NodeHolder> NodeHolder for &H {
+       type CM = H::CM;
+       fn node(&self) -> &ChannelManager<
+               <Self::CM as AChannelManager>::M,
+               <Self::CM as AChannelManager>::T,
+               <Self::CM as AChannelManager>::ES,
+               <Self::CM as AChannelManager>::NS,
+               <Self::CM as AChannelManager>::SP,
+               <Self::CM as AChannelManager>::F,
+               <Self::CM as AChannelManager>::R,
+               <Self::CM as AChannelManager>::L> { (*self).node() }
+       fn chain_monitor(&self) -> Option<&test_utils::TestChainMonitor> { (*self).chain_monitor() }
+}
+impl<'a, 'b: 'a, 'c: 'b> NodeHolder for Node<'a, 'b, 'c> {
+       type CM = TestChannelManager<'a, 'b, 'c>;
+       fn node(&self) -> &TestChannelManager<'a, 'b, 'c> { &self.node }
+       fn chain_monitor(&self) -> Option<&test_utils::TestChainMonitor> { Some(self.chain_monitor) }
+}
+
 impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
        fn drop(&mut self) {
                if !panicking() {
@@ -486,36 +545,27 @@ pub fn create_chan_between_nodes_with_value<'a, 'b, 'c, 'd>(node_a: &'a Node<'b,
 }
 
 /// Gets an RAA and CS which were sent in response to a commitment update
-///
-/// Should only be used directly when the `$node` is not actually a [`Node`].
-macro_rules! do_get_revoke_commit_msgs {
-       ($node: expr, $recipient: expr) => { {
-               let events = $node.node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), 2);
-               (match events[0] {
-                       MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
-                               assert_eq!(node_id, $recipient);
-                               (*msg).clone()
-                       },
-                       _ => panic!("Unexpected event"),
-               }, match events[1] {
-                       MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
-                               assert_eq!(node_id, $recipient);
-                               assert!(updates.update_add_htlcs.is_empty());
-                               assert!(updates.update_fulfill_htlcs.is_empty());
-                               assert!(updates.update_fail_htlcs.is_empty());
-                               assert!(updates.update_fail_malformed_htlcs.is_empty());
-                               assert!(updates.update_fee.is_none());
-                               updates.commitment_signed.clone()
-                       },
-                       _ => panic!("Unexpected event"),
-               })
-       } }
-}
-
-/// Gets an RAA and CS which were sent in response to a commitment update
-pub fn get_revoke_commit_msgs(node: &Node, recipient: &PublicKey) -> (msgs::RevokeAndACK, msgs::CommitmentSigned) {
-       do_get_revoke_commit_msgs!(node, recipient)
+pub fn get_revoke_commit_msgs<CM: AChannelManager, H: NodeHolder<CM=CM>>(node: &H, recipient: &PublicKey) -> (msgs::RevokeAndACK, msgs::CommitmentSigned) {
+       let events = node.node().get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 2);
+       (match events[0] {
+               MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
+                       assert_eq!(node_id, recipient);
+                       (*msg).clone()
+               },
+               _ => panic!("Unexpected event"),
+       }, match events[1] {
+               MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
+                       assert_eq!(node_id, recipient);
+                       assert!(updates.update_add_htlcs.is_empty());
+                       assert!(updates.update_fulfill_htlcs.is_empty());
+                       assert!(updates.update_fail_htlcs.is_empty());
+                       assert!(updates.update_fail_malformed_htlcs.is_empty());
+                       assert!(updates.update_fee.is_none());
+                       updates.commitment_signed.clone()
+               },
+               _ => panic!("Unexpected event"),
+       })
 }
 
 #[macro_export]
@@ -774,10 +824,12 @@ macro_rules! unwrap_send_err {
 }
 
 /// Check whether N channel monitor(s) have been added.
-pub fn check_added_monitors(node: &Node, count: usize) {
-       let mut added_monitors = node.chain_monitor.added_monitors.lock().unwrap();
-       assert_eq!(added_monitors.len(), count);
-       added_monitors.clear();
+pub fn check_added_monitors<CM: AChannelManager, H: NodeHolder<CM=CM>>(node: &H, count: usize) {
+       if let Some(chain_monitor) = node.chain_monitor() {
+               let mut added_monitors = chain_monitor.added_monitors.lock().unwrap();
+               assert_eq!(added_monitors.len(), count);
+               added_monitors.clear();
+       }
 }
 
 /// Check whether N channel monitor(s) have been added.
@@ -883,6 +935,7 @@ pub fn sign_funding_transaction<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &
                assert_eq!(added_monitors[0].0, funding_output);
                added_monitors.clear();
        }
+       expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
 
        node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id()));
        {
@@ -891,6 +944,7 @@ pub fn sign_funding_transaction<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &
                assert_eq!(added_monitors[0].0, funding_output);
                added_monitors.clear();
        }
+       expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
 
        let events_4 = node_a.node.get_and_clear_pending_events();
        assert_eq!(events_4.len(), 0);
@@ -942,6 +996,8 @@ pub fn open_zero_conf_channel<'a, 'b, 'c, 'd>(initiator: &'a Node<'b, 'c, 'd>, r
                MessageSendEvent::SendFundingSigned { node_id, msg } => {
                        assert_eq!(*node_id, initiator.node.get_our_node_id());
                        initiator.node.handle_funding_signed(&receiver.node.get_our_node_id(), &msg);
+                       expect_channel_pending_event(&initiator, &receiver.node.get_our_node_id());
+                       expect_channel_pending_event(&receiver, &initiator.node.get_our_node_id());
                        check_added_monitors!(initiator, 1);
 
                        assert_eq!(initiator.tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
@@ -955,11 +1011,13 @@ pub fn open_zero_conf_channel<'a, 'b, 'c, 'd>(initiator: &'a Node<'b, 'c, 'd>, r
                MessageSendEvent::SendChannelReady { node_id, msg } => {
                        assert_eq!(*node_id, initiator.node.get_our_node_id());
                        initiator.node.handle_channel_ready(&receiver.node.get_our_node_id(), &msg);
+                       expect_channel_ready_event(&initiator, &receiver.node.get_our_node_id());
                }
                _ => panic!("Unexpected event"),
        }
 
        receiver.node.handle_channel_ready(&initiator.node.get_our_node_id(), &as_channel_ready);
+       expect_channel_ready_event(&receiver, &initiator.node.get_our_node_id());
 
        let as_channel_update = get_event_msg!(initiator, MessageSendEvent::SendChannelUpdate, receiver.node.get_our_node_id());
        let bs_channel_update = get_event_msg!(receiver, MessageSendEvent::SendChannelUpdate, initiator.node.get_our_node_id());
@@ -970,9 +1028,6 @@ pub fn open_zero_conf_channel<'a, 'b, 'c, 'd>(initiator: &'a Node<'b, 'c, 'd>, r
        assert_eq!(initiator.node.list_usable_channels().len(), initiator_channels + 1);
        assert_eq!(receiver.node.list_usable_channels().len(), receiver_channels + 1);
 
-       expect_channel_ready_event(&initiator, &receiver.node.get_our_node_id());
-       expect_channel_ready_event(&receiver, &initiator.node.get_our_node_id());
-
        (tx, as_channel_ready.channel_id)
 }
 
@@ -1097,7 +1152,10 @@ pub fn create_unannounced_chan_between_nodes_with_value<'a, 'b, 'c, 'd>(nodes: &
        check_added_monitors!(nodes[b], 1);
 
        let cs_funding_signed = get_event_msg!(nodes[b], MessageSendEvent::SendFundingSigned, nodes[a].node.get_our_node_id());
+       expect_channel_pending_event(&nodes[b], &nodes[a].node.get_our_node_id());
+
        nodes[a].node.handle_funding_signed(&nodes[b].node.get_our_node_id(), &cs_funding_signed);
+       expect_channel_pending_event(&nodes[a], &nodes[b].node.get_our_node_id());
        check_added_monitors!(nodes[a], 1);
 
        assert_eq!(nodes[a].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
@@ -1521,23 +1579,30 @@ macro_rules! commitment_signed_dance {
                        bs_revoke_and_ack
                }
        };
-       ($node_a: expr, $node_b: expr, (), $fail_backwards: expr, true /* skip last step */, true /* return extra message */) => {
-               {
-                       let (extra_msg_option, bs_revoke_and_ack) = $crate::ln::functional_test_utils::do_main_commitment_signed_dance(&$node_a, &$node_b, $fail_backwards);
-                       $node_a.node.handle_revoke_and_ack(&$node_b.node.get_our_node_id(), &bs_revoke_and_ack);
-                       $crate::ln::functional_test_utils::check_added_monitors(&$node_a, 1);
-                       extra_msg_option
-               }
-       };
        ($node_a: expr, $node_b: expr, (), $fail_backwards: expr, true /* skip last step */, false /* no extra message */) => {
-               assert!(commitment_signed_dance!($node_a, $node_b, (), $fail_backwards, true, true).is_none());
+               assert!($crate::ln::functional_test_utils::commitment_signed_dance_through_cp_raa(&$node_a, &$node_b, $fail_backwards).is_none());
        };
        ($node_a: expr, $node_b: expr, $commitment_signed: expr, $fail_backwards: expr) => {
                $crate::ln::functional_test_utils::do_commitment_signed_dance(&$node_a, &$node_b, &$commitment_signed, $fail_backwards, false);
        }
 }
 
-
+/// Runs the commitment_signed dance after the initial commitment_signed is delivered through to
+/// the initiator's `revoke_and_ack` response. i.e. [`do_main_commitment_signed_dance`] plus the
+/// `revoke_and_ack` response to it.
+///
+/// Returns any additional message `node_b` generated in addition to the `revoke_and_ack` response.
+pub fn commitment_signed_dance_through_cp_raa(node_a: &Node<'_, '_, '_>, node_b: &Node<'_, '_, '_>, fail_backwards: bool) -> Option<MessageSendEvent> {
+       let (extra_msg_option, bs_revoke_and_ack) = do_main_commitment_signed_dance(node_a, node_b, fail_backwards);
+       node_a.node.handle_revoke_and_ack(&node_b.node.get_our_node_id(), &bs_revoke_and_ack);
+       check_added_monitors(node_a, 1);
+       extra_msg_option
+}
+
+/// Does the main logic in the commitment_signed dance. After the first `commitment_signed` has
+/// been delivered, this method picks up and delivers the response `revoke_and_ack` and
+/// `commitment_signed`, returning the recipient's `revoke_and_ack` and any extra message it may
+/// have included.
 pub fn do_main_commitment_signed_dance(node_a: &Node<'_, '_, '_>, node_b: &Node<'_, '_, '_>, fail_backwards: bool) -> (Option<MessageSendEvent>, msgs::RevokeAndACK) {
        let (as_revoke_and_ack, as_commitment_signed) = get_revoke_commit_msgs!(node_a, node_b.node.get_our_node_id());
        check_added_monitors!(node_b, 0);
@@ -1566,6 +1631,11 @@ pub fn do_main_commitment_signed_dance(node_a: &Node<'_, '_, '_>, node_b: &Node<
        (extra_msg_option, bs_revoke_and_ack)
 }
 
+/// Runs a full commitment_signed dance, delivering a commitment_signed, the responding
+/// `revoke_and_ack` and `commitment_signed`, and then the final `revoke_and_ack` response.
+///
+/// If `skip_last_step` is unset, also checks for the payment failure update for the previous hop
+/// on failure or that no new messages are left over on success.
 pub fn do_commitment_signed_dance(node_a: &Node<'_, '_, '_>, node_b: &Node<'_, '_, '_>, commitment_signed: &msgs::CommitmentSigned, fail_backwards: bool, skip_last_step: bool) {
        check_added_monitors!(node_a, 0);
        assert!(node_a.node.get_and_clear_pending_msg_events().is_empty());
@@ -1674,7 +1744,7 @@ macro_rules! expect_payment_claimable {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       $crate::events::Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id: _, via_user_channel_id: _ } => {
+                       $crate::events::Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, .. } => {
                                assert_eq!($expected_payment_hash, *payment_hash);
                                assert_eq!($expected_recv_value, amount_msat);
                                assert_eq!($expected_receiver_node_id, receiver_node_id.unwrap());
@@ -1707,6 +1777,44 @@ macro_rules! expect_payment_claimed {
        }
 }
 
+pub fn expect_payment_sent<CM: AChannelManager, H: NodeHolder<CM=CM>>(node: &H,
+       expected_payment_preimage: PaymentPreimage, expected_fee_msat_opt: Option<Option<u64>>,
+       expect_per_path_claims: bool,
+) {
+       let events = node.node().get_and_clear_pending_events();
+       let expected_payment_hash = PaymentHash(
+               bitcoin::hashes::sha256::Hash::hash(&expected_payment_preimage.0).into_inner());
+       if expect_per_path_claims {
+               assert!(events.len() > 1);
+       } else {
+               assert_eq!(events.len(), 1);
+       }
+       let expected_payment_id = match events[0] {
+               Event::PaymentSent { ref payment_id, ref payment_preimage, ref payment_hash, ref fee_paid_msat } => {
+                       assert_eq!(expected_payment_preimage, *payment_preimage);
+                       assert_eq!(expected_payment_hash, *payment_hash);
+                       if let Some(expected_fee_msat) = expected_fee_msat_opt {
+                               assert_eq!(*fee_paid_msat, expected_fee_msat);
+                       } else {
+                               assert!(fee_paid_msat.is_some());
+                       }
+                       payment_id.unwrap()
+               },
+               _ => panic!("Unexpected event"),
+       };
+       if expect_per_path_claims {
+               for i in 1..events.len() {
+                       match events[i] {
+                               Event::PaymentPathSuccessful { payment_id, payment_hash, .. } => {
+                                       assert_eq!(payment_id, expected_payment_id);
+                                       assert_eq!(payment_hash, Some(expected_payment_hash));
+                               },
+                               _ => panic!("Unexpected event"),
+                       }
+               }
+       }
+}
+
 #[cfg(test)]
 #[macro_export]
 macro_rules! expect_payment_sent_without_paths {
@@ -1726,40 +1834,10 @@ macro_rules! expect_payment_sent {
        ($node: expr, $expected_payment_preimage: expr, $expected_fee_msat_opt: expr) => {
                $crate::expect_payment_sent!($node, $expected_payment_preimage, $expected_fee_msat_opt, true);
        };
-       ($node: expr, $expected_payment_preimage: expr, $expected_fee_msat_opt: expr, $expect_paths: expr) => { {
-               use bitcoin::hashes::Hash as _;
-               let events = $node.node.get_and_clear_pending_events();
-               let expected_payment_hash = $crate::ln::PaymentHash(
-                       bitcoin::hashes::sha256::Hash::hash(&$expected_payment_preimage.0).into_inner());
-               if $expect_paths {
-                       assert!(events.len() > 1);
-               } else {
-                       assert_eq!(events.len(), 1);
-               }
-               let expected_payment_id = match events[0] {
-                       $crate::events::Event::PaymentSent { ref payment_id, ref payment_preimage, ref payment_hash, ref fee_paid_msat } => {
-                               assert_eq!($expected_payment_preimage, *payment_preimage);
-                               assert_eq!(expected_payment_hash, *payment_hash);
-                               assert!(fee_paid_msat.is_some());
-                               if $expected_fee_msat_opt.is_some() {
-                                       assert_eq!(*fee_paid_msat, $expected_fee_msat_opt);
-                               }
-                               payment_id.unwrap()
-                       },
-                       _ => panic!("Unexpected event"),
-               };
-               if $expect_paths {
-                       for i in 1..events.len() {
-                               match events[i] {
-                                       $crate::events::Event::PaymentPathSuccessful { payment_id, payment_hash, .. } => {
-                                               assert_eq!(payment_id, expected_payment_id);
-                                               assert_eq!(payment_hash, Some(expected_payment_hash));
-                                       },
-                                       _ => panic!("Unexpected event"),
-                               }
-                       }
-               }
-       } }
+       ($node: expr, $expected_payment_preimage: expr, $expected_fee_msat_opt: expr, $expect_paths: expr) => {
+               $crate::ln::functional_test_utils::expect_payment_sent(&$node, $expected_payment_preimage,
+                       $expected_fee_msat_opt.map(|o| Some(o)), $expect_paths);
+       }
 }
 
 #[cfg(test)]
@@ -1801,6 +1879,18 @@ macro_rules! expect_payment_forwarded {
        }
 }
 
+#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+pub fn expect_channel_pending_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, expected_counterparty_node_id: &PublicKey) {
+       let events = node.node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 1);
+       match events[0] {
+               crate::events::Event::ChannelPending { ref counterparty_node_id, .. } => {
+                       assert_eq!(*expected_counterparty_node_id, *counterparty_node_id);
+               },
+               _ => panic!("Unexpected event"),
+       }
+}
+
 #[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
 pub fn expect_channel_ready_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, expected_counterparty_node_id: &PublicKey) {
        let events = node.node.get_and_clear_pending_events();
@@ -1813,7 +1903,6 @@ pub fn expect_channel_ready_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, ex
        }
 }
 
-
 pub struct PaymentFailedConditions<'a> {
        pub(crate) expected_htlc_error_data: Option<(u16, &'a [u8])>,
        pub(crate) expected_blamed_scid: Option<u64>,
@@ -1920,9 +2009,14 @@ pub fn expect_payment_failed_conditions_event<'a, 'b, 'c, 'd, 'e>(
        };
        if !conditions.expected_mpp_parts_remain {
                match &payment_failed_events[1] {
-                       Event::PaymentFailed { ref payment_hash, ref payment_id } => {
+                       Event::PaymentFailed { ref payment_hash, ref payment_id, ref reason } => {
                                assert_eq!(*payment_hash, expected_payment_hash, "unexpected second payment_hash");
                                assert_eq!(*payment_id, expected_payment_id);
+                               assert_eq!(reason.unwrap(), if expected_payment_failed_permanently {
+                                       PaymentFailureReason::RecipientRejected
+                               } else {
+                                       PaymentFailureReason::RetriesExhausted
+                               });
                        }
                        _ => panic!("Unexpected second event"),
                }
@@ -1939,15 +2033,17 @@ pub fn expect_payment_failed_conditions<'a, 'b, 'c, 'd, 'e>(
 
 pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) -> PaymentId {
        let payment_id = PaymentId(origin_node.keys_manager.backing.get_secure_random_bytes());
-       origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), payment_id).unwrap();
+       origin_node.node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), payment_id).unwrap();
        check_added_monitors!(origin_node, expected_paths.len());
        pass_along_route(origin_node, expected_paths, recv_value, our_payment_hash, our_payment_secret);
        payment_id
 }
 
-pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) {
+pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) -> Option<Event> {
        let mut payment_event = SendEvent::from_event(ev);
        let mut prev_node = origin_node;
+       let mut event = None;
 
        for (idx, &node) in expected_path.iter().enumerate() {
                assert_eq!(node.node.get_our_node_id(), payment_event.node_id);
@@ -1962,26 +2058,33 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
                        let events_2 = node.node.get_and_clear_pending_events();
                        if payment_claimable_expected {
                                assert_eq!(events_2.len(), 1);
-                               match events_2[0] {
-                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, ref via_channel_id, ref via_user_channel_id } => {
+                               match &events_2[0] {
+                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat,
+                                               receiver_node_id, ref via_channel_id, ref via_user_channel_id,
+                                               claim_deadline, onion_fields,
+                                       } => {
                                                assert_eq!(our_payment_hash, *payment_hash);
                                                assert_eq!(node.node.get_our_node_id(), receiver_node_id.unwrap());
+                                               assert!(onion_fields.is_some());
                                                match &purpose {
                                                        PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                                assert_eq!(expected_preimage, *payment_preimage);
                                                                assert_eq!(our_payment_secret.unwrap(), *payment_secret);
+                                                               assert_eq!(Some(*payment_secret), onion_fields.as_ref().unwrap().payment_secret);
                                                        },
                                                        PaymentPurpose::SpontaneousPayment(payment_preimage) => {
                                                                assert_eq!(expected_preimage.unwrap(), *payment_preimage);
                                                                assert!(our_payment_secret.is_none());
                                                        },
                                                }
-                                               assert_eq!(amount_msat, recv_value);
+                                               assert_eq!(*amount_msat, recv_value);
                                                assert!(node.node.list_channels().iter().any(|details| details.channel_id == via_channel_id.unwrap()));
                                                assert!(node.node.list_channels().iter().any(|details| details.user_channel_id == via_user_channel_id.unwrap()));
+                                               assert!(claim_deadline.unwrap() > node.best_block_info().1);
                                        },
                                        _ => panic!("Unexpected event"),
                                }
+                               event = Some(events_2[0].clone());
                        } else {
                                assert!(events_2.is_empty());
                        }
@@ -1995,10 +2098,11 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
 
                prev_node = node;
        }
+       event
 }
 
-pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, expected_preimage: Option<PaymentPreimage>) {
-       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_claimable_expected, true, expected_preimage);
+pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, expected_preimage: Option<PaymentPreimage>) -> Option<Event> {
+       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_claimable_expected, true, expected_preimage)
 }
 
 pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
@@ -2171,8 +2275,8 @@ pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route:
                .with_features(expected_route.last().unwrap().node.invoice_features());
        let route = get_route(origin_node, &payment_params, recv_value, TEST_FINAL_CLTV).unwrap();
        assert_eq!(route.paths.len(), 1);
-       assert_eq!(route.paths[0].len(), expected_route.len());
-       for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
+       assert_eq!(route.paths[0].hops.len(), expected_route.len());
+       for (node, hop) in expected_route.iter().zip(route.paths[0].hops.iter()) {
                assert_eq!(hop.pubkey, node.node.get_our_node_id());
        }
 
@@ -2192,13 +2296,15 @@ pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_rou
                &origin_node.node.get_our_node_id(), &payment_params, &network_graph,
                None, recv_value, TEST_FINAL_CLTV, origin_node.logger, &scorer, &random_seed_bytes).unwrap();
        assert_eq!(route.paths.len(), 1);
-       assert_eq!(route.paths[0].len(), expected_route.len());
-       for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
+       assert_eq!(route.paths[0].hops.len(), expected_route.len());
+       for (node, hop) in expected_route.iter().zip(route.paths[0].hops.iter()) {
                assert_eq!(hop.pubkey, node.node.get_our_node_id());
        }
 
-       let (_, our_payment_hash, our_payment_preimage) = get_payment_preimage_hash!(expected_route.last().unwrap());
-       unwrap_send_err!(origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_preimage), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(expected_route.last().unwrap());
+       unwrap_send_err!(origin_node.node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)),
+               true, APIError::ChannelUnavailable { ref err },
                assert!(err.contains("Cannot send value that would put us over the max HTLC value in flight our peer will accept")));
 }
 
@@ -2216,10 +2322,10 @@ pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
        let expected_destinations: Vec<HTLCDestination> = repeat(HTLCDestination::FailedPayment { payment_hash: our_payment_hash }).take(expected_paths.len()).collect();
        expect_pending_htlcs_forwardable_and_htlc_handling_failed!(expected_paths[0].last().unwrap(), expected_destinations);
 
-       pass_failed_payment_back(origin_node, expected_paths, skip_last, our_payment_hash);
+       pass_failed_payment_back(origin_node, expected_paths, skip_last, our_payment_hash, PaymentFailureReason::RecipientRejected);
 }
 
-pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash) {
+pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash, expected_fail_reason: PaymentFailureReason) {
        let mut expected_paths: Vec<_> = expected_paths_slice.iter().collect();
        check_added_monitors!(expected_paths[0].last().unwrap(), expected_paths.len());
 
@@ -2297,7 +2403,7 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
                                        assert_eq!(payment_hash, our_payment_hash);
                                        assert!(payment_failed_permanently);
                                        for (idx, hop) in expected_route.iter().enumerate() {
-                                               assert_eq!(hop.node.get_our_node_id(), path[idx].pubkey);
+                                               assert_eq!(hop.node.get_our_node_id(), path.hops[idx].pubkey);
                                        }
                                        payment_id.unwrap()
                                },
@@ -2305,9 +2411,10 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
                        };
                        if i == expected_paths.len() - 1 {
                                match events[1] {
-                                       Event::PaymentFailed { ref payment_hash, ref payment_id } => {
+                                       Event::PaymentFailed { ref payment_hash, ref payment_id, ref reason } => {
                                                assert_eq!(*payment_hash, our_payment_hash, "unexpected second payment_hash");
                                                assert_eq!(*payment_id, expected_payment_id);
+                                               assert_eq!(reason.unwrap(), expected_fail_reason);
                                        }
                                        _ => panic!("Unexpected second event"),
                                }
@@ -2329,10 +2436,7 @@ pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &
 pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
        let mut chan_mon_cfgs = Vec::new();
        for i in 0..node_count {
-               let tx_broadcaster = test_utils::TestBroadcaster {
-                       txn_broadcasted: Mutex::new(Vec::new()),
-                       blocks: Arc::new(Mutex::new(vec![(genesis_block(Network::Testnet), 0)])),
-               };
+               let tx_broadcaster = test_utils::TestBroadcaster::new(Network::Testnet);
                let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
                let chain_source = test_utils::TestChainSource::new(Network::Testnet);
                let logger = test_utils::TestLogger::with_id(format!("node {}", i));
@@ -2452,6 +2556,8 @@ pub enum HTLCType { NONE, TIMEOUT, SUCCESS }
 /// also fail.
 pub fn test_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, chan: &(msgs::ChannelUpdate, msgs::ChannelUpdate, [u8; 32], Transaction), commitment_tx: Option<Transaction>, has_htlc_tx: HTLCType) -> Vec<Transaction>  {
        let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let mut txn_seen = HashSet::new();
+       node_txn.retain(|tx| txn_seen.insert(tx.txid()));
        assert!(node_txn.len() >= if commitment_tx.is_some() { 0 } else { 1 } + if has_htlc_tx == HTLCType::NONE { 0 } else { 1 });
 
        let mut res = Vec::with_capacity(2);
@@ -2515,22 +2621,23 @@ pub fn test_revoked_htlc_claim_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>
 
 pub fn check_preimage_claim<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, prev_txn: &Vec<Transaction>) -> Vec<Transaction>  {
        let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let mut txn_seen = HashSet::new();
+       node_txn.retain(|tx| txn_seen.insert(tx.txid()));
 
-       assert!(node_txn.len() >= 1);
-       assert_eq!(node_txn[0].input.len(), 1);
        let mut found_prev = false;
-
-       for tx in prev_txn {
-               if node_txn[0].input[0].previous_output.txid == tx.txid() {
-                       check_spends!(node_txn[0], tx);
-                       let mut iter = node_txn[0].input[0].witness.iter();
-                       iter.next().expect("expected 3 witness items");
-                       iter.next().expect("expected 3 witness items");
-                       assert!(iter.next().expect("expected 3 witness items").len() > 106); // must spend an htlc output
-                       assert_eq!(tx.input.len(), 1); // must spend a commitment tx
-
-                       found_prev = true;
-                       break;
+       for prev_tx in prev_txn {
+               for tx in &*node_txn {
+                       if tx.input[0].previous_output.txid == prev_tx.txid() {
+                               check_spends!(tx, prev_tx);
+                               let mut iter = tx.input[0].witness.iter();
+                               iter.next().expect("expected 3 witness items");
+                               iter.next().expect("expected 3 witness items");
+                               assert!(iter.next().expect("expected 3 witness items").len() > 106); // must spend an htlc output
+                               assert_eq!(tx.input.len(), 1); // must spend a commitment tx
+
+                               found_prev = true;
+                               break;
+                       }
                }
        }
        assert!(found_prev);
@@ -2646,10 +2753,9 @@ macro_rules! handle_chan_reestablish_msgs {
                        }
 
                        let mut had_channel_update = false; // ChannelUpdate may be now or later, but not both
-                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, .. }) = msg_events.get(idx) {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
-                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
                                had_channel_update = true;
                        }
 
@@ -2693,10 +2799,9 @@ macro_rules! handle_chan_reestablish_msgs {
                                }
                        }
 
-                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, .. }) = msg_events.get(idx) {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
-                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
                                assert!(!had_channel_update);
                        }
 
index 56586b0a29c569be57700a316e109e1bb76edb24..2b8c8b8e3414af084180668fcc70b0ad59409ee8 100644 (file)
@@ -18,15 +18,15 @@ use crate::chain::channelmonitor;
 use crate::chain::channelmonitor::{CLTV_CLAIM_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS, ANTI_REORG_DELAY};
 use crate::chain::transaction::OutPoint;
 use crate::chain::keysinterface::{ChannelSigner, EcdsaChannelSigner, EntropySource};
-use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose, ClosureReason, HTLCDestination};
+use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentPurpose, ClosureReason, HTLCDestination, PaymentFailureReason};
 use crate::ln::{PaymentPreimage, PaymentSecret, PaymentHash};
 use crate::ln::channel::{commitment_tx_base_weight, COMMITMENT_TX_WEIGHT_PER_HTLC, CONCURRENT_INBOUND_HTLC_FEE_BUFFER, FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE, MIN_AFFORDABLE_HTLC_COUNT};
-use crate::ln::channelmanager::{self, PaymentId, RAACommitmentOrder, PaymentSendFailure, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA};
+use crate::ln::channelmanager::{self, PaymentId, RAACommitmentOrder, PaymentSendFailure, RecipientOnionFields, BREAKDOWN_TIMEOUT, ENABLE_GOSSIP_TICKS, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA};
 use crate::ln::channel::{Channel, ChannelError};
 use crate::ln::{chan_utils, onion_utils};
 use crate::ln::chan_utils::{OFFERED_HTLC_SCRIPT_WEIGHT, htlc_success_tx_weight, htlc_timeout_tx_weight, HTLCOutputInCommitment};
 use crate::routing::gossip::{NetworkGraph, NetworkUpdate};
-use crate::routing::router::{PaymentParameters, Route, RouteHop, RouteParameters, find_route, get_route};
+use crate::routing::router::{Path, PaymentParameters, Route, RouteHop, RouteParameters, find_route, get_route};
 use crate::ln::features::{ChannelFeatures, NodeFeatures};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
@@ -257,7 +257,8 @@ fn test_async_inbound_update_fee() {
 
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 40000);
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -356,7 +357,8 @@ fn test_update_fee_unordered_raa() {
 
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 40000);
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -541,6 +543,8 @@ fn do_test_sanity_on_in_flight_opens(steps: u8) {
                assert_eq!(added_monitors[0].0, funding_output);
                added_monitors.clear();
        }
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
+
        let funding_signed = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
 
        if steps & 0x0f == 5 { return; }
@@ -552,6 +556,7 @@ fn do_test_sanity_on_in_flight_opens(steps: u8) {
                added_monitors.clear();
        }
 
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
        let events_4 = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events_4.len(), 0);
 
@@ -735,7 +740,9 @@ fn test_update_fee_that_funder_cannot_afford() {
        let commit_signed_msg = msgs::CommitmentSigned {
                channel_id: chan.2,
                signature: res.0,
-               htlc_signatures: res.1
+               htlc_signatures: res.1,
+               #[cfg(taproot)]
+               partial_signature_with_nonce: None,
        };
 
        let update_fee = msgs::UpdateFee {
@@ -789,7 +796,8 @@ fn test_update_fee_with_fundee_update_add_htlc() {
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 800000);
 
        // nothing happens since node[1] is in AwaitingRemoteRevoke
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        {
                let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                assert_eq!(added_monitors.len(), 0);
@@ -1036,7 +1044,7 @@ fn fake_network_test() {
        });
        hops[1].fee_msat = chan_4.1.contents.fee_base_msat as u64 + chan_4.1.contents.fee_proportional_millionths as u64 * hops[2].fee_msat as u64 / 1000000;
        hops[0].fee_msat = chan_3.0.contents.fee_base_msat as u64 + chan_3.0.contents.fee_proportional_millionths as u64 * hops[1].fee_msat as u64 / 1000000;
-       let payment_preimage_1 = send_along_route(&nodes[1], Route { paths: vec![hops], payment_params: None }, &vec!(&nodes[2], &nodes[3], &nodes[1])[..], 1000000).0;
+       let payment_preimage_1 = send_along_route(&nodes[1], Route { paths: vec![Path { hops, blinded_tail: None }], payment_params: None }, &vec!(&nodes[2], &nodes[3], &nodes[1])[..], 1000000).0;
 
        let mut hops = Vec::with_capacity(3);
        hops.push(RouteHop {
@@ -1065,7 +1073,7 @@ fn fake_network_test() {
        });
        hops[1].fee_msat = chan_2.1.contents.fee_base_msat as u64 + chan_2.1.contents.fee_proportional_millionths as u64 * hops[2].fee_msat as u64 / 1000000;
        hops[0].fee_msat = chan_3.1.contents.fee_base_msat as u64 + chan_3.1.contents.fee_proportional_millionths as u64 * hops[1].fee_msat as u64 / 1000000;
-       let payment_hash_2 = send_along_route(&nodes[1], Route { paths: vec![hops], payment_params: None }, &vec!(&nodes[3], &nodes[2], &nodes[1])[..], 1000000).1;
+       let payment_hash_2 = send_along_route(&nodes[1], Route { paths: vec![Path { hops, blinded_tail: None }], payment_params: None }, &vec!(&nodes[3], &nodes[2], &nodes[1])[..], 1000000).1;
 
        // Claim the rebalances...
        fail_payment(&nodes[1], &vec!(&nodes[3], &nodes[2], &nodes[1])[..], payment_hash_2);
@@ -1099,9 +1107,10 @@ fn holding_cell_htlc_counting() {
        let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2);
 
        let mut payments = Vec::new();
-       for _ in 0..crate::ln::channel::OUR_MAX_HTLCS {
+       for _ in 0..50 {
                let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[1].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                payments.push((payment_preimage, payment_hash));
        }
        check_added_monitors!(nodes[1], 1);
@@ -1116,7 +1125,9 @@ fn holding_cell_htlc_counting() {
        // another HTLC.
        let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
        {
-               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[1].node.send_payment_with_route(&route, payment_hash_1,
+                               RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
                nodes[1].logger.assert_log_contains("lightning::ln::channelmanager", "Cannot push more than their max accepted HTLCs", 1);
@@ -1125,7 +1136,8 @@ fn holding_cell_htlc_counting() {
        // This should also be true if we try to forward a payment.
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 100000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1272,7 +1284,7 @@ fn test_duplicate_htlc_different_direction_onchain() {
        mine_transaction(&nodes[0], &remote_txn[0]);
        check_added_monitors!(nodes[0], 1);
        check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
 
        let claim_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
        assert_eq!(claim_txn.len(), 3);
@@ -1335,7 +1347,8 @@ fn test_basic_channel_reserve() {
        let commit_tx_fee = 2 * commit_tx_fee_msat(get_feerate!(nodes[0], nodes[1], chan.2), 1 + 1, get_opt_anchors!(nodes[0], nodes[1], chan.2));
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send + 1);
-       let err = nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).err().unwrap();
+       let err = nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).err().unwrap();
        match err {
                PaymentSendFailure::AllFailedResendSafe(ref fails) => {
                        match &fails[0] {
@@ -1368,7 +1381,8 @@ fn test_fee_spike_violation_fails_htlc() {
        let cur_height = nodes[1].node.best_block.read().unwrap().height() + 1;
 
        let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
-       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 3460001, &Some(payment_secret), cur_height, &None).unwrap();
+       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0],
+               3460001, RecipientOnionFields::secret_only(payment_secret), cur_height, &None).unwrap();
        let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
        let msg = msgs::UpdateAddHTLC {
                channel_id: chan.2,
@@ -1455,7 +1469,9 @@ fn test_fee_spike_violation_fails_htlc() {
        let commit_signed_msg = msgs::CommitmentSigned {
                channel_id: chan.2,
                signature: res.0,
-               htlc_signatures: res.1
+               htlc_signatures: res.1,
+               #[cfg(taproot)]
+               partial_signature_with_nonce: None,
        };
 
        // Send the commitment_signed message to the nodes[1].
@@ -1466,7 +1482,9 @@ fn test_fee_spike_violation_fails_htlc() {
        let raa_msg = msgs::RevokeAndACK {
                channel_id: chan.2,
                per_commitment_secret: local_secret,
-               next_per_commitment_point: next_local_point
+               next_per_commitment_point: next_local_point,
+               #[cfg(taproot)]
+               next_local_nonce: None,
        };
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &raa_msg);
 
@@ -1514,7 +1532,9 @@ fn test_chan_reserve_violation_outbound_htlc_inbound_chan() {
 
        // However one more HTLC should be significantly over the reserve amount and fail.
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 1_000_000);
-       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[1].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send value that would put counterparty balance under holder-announced channel reserve value"));
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Cannot send value that would put counterparty balance under holder-announced channel reserve value".to_string(), 1);
@@ -1549,7 +1569,8 @@ fn test_chan_reserve_violation_inbound_htlc_outbound_channel() {
        let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
        let cur_height = nodes[1].node.best_block.read().unwrap().height() + 1;
        let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
-       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 700_000, &Some(payment_secret), cur_height, &None).unwrap();
+       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0],
+               700_000, RecipientOnionFields::secret_only(payment_secret), cur_height, &None).unwrap();
        let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
        let msg = msgs::UpdateAddHTLC {
                channel_id: chan.2,
@@ -1605,7 +1626,9 @@ fn test_chan_reserve_dust_inbound_htlcs_outbound_chan() {
 
        // One more than the dust amt should fail, however.
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], dust_amt + 1);
-       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[1].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send value that would put counterparty balance under holder-announced channel reserve value"));
 }
 
@@ -1699,7 +1722,8 @@ fn test_chan_reserve_violation_inbound_htlc_inbound_chan() {
        // Add a pending HTLC.
        let (route_1, our_payment_hash_1, _, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat_1);
        let payment_event_1 = {
-               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route_1, our_payment_hash_1,
+                       RecipientOnionFields::secret_only(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1719,7 +1743,8 @@ fn test_chan_reserve_violation_inbound_htlc_inbound_chan() {
        let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
        let cur_height = nodes[0].node.best_block.read().unwrap().height() + 1;
        let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route_2.paths[0], &session_priv).unwrap();
-       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route_2.paths[0], recv_value_2, &None, cur_height, &None).unwrap();
+       let (onion_payloads, htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(
+               &route_2.paths[0], recv_value_2, RecipientOnionFields::spontaneous_empty(), cur_height, &None).unwrap();
        let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &our_payment_hash_1);
        let msg = msgs::UpdateAddHTLC {
                channel_id: chan.2,
@@ -1806,10 +1831,12 @@ fn test_channel_reserve_holding_cell_htlcs() {
                let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)
                        .with_features(nodes[2].node.invoice_features()).with_max_channel_saturation_power_of_half(0);
                let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, recv_value_0, TEST_FINAL_CLTV);
-               route.paths[0].last_mut().unwrap().fee_msat += 1;
-               assert!(route.paths[0].iter().rev().skip(1).all(|h| h.fee_msat == feemsat));
+               route.paths[0].hops.last_mut().unwrap().fee_msat += 1;
+               assert!(route.paths[0].hops.iter().rev().skip(1).all(|h| h.fee_msat == feemsat));
 
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put us over the max HTLC value in flight our peer will accept \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
                nodes[0].logger.assert_log_contains("lightning::ln::channelmanager", "Cannot send value that would put us over the max HTLC value in flight our peer will accept", 1);
@@ -1865,7 +1892,8 @@ fn test_channel_reserve_holding_cell_htlcs() {
 
        let (route_1, our_payment_hash_1, our_payment_preimage_1, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_1);
        let payment_event_1 = {
-               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route_1, our_payment_hash_1,
+                       RecipientOnionFields::secret_only(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1878,7 +1906,9 @@ fn test_channel_reserve_holding_cell_htlcs() {
        let recv_value_2 = stat01.value_to_self_msat - amt_msat_1 - stat01.channel_reserve_msat - total_fee_msat - commit_tx_fee_2_htlcs;
        {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_2 + 1);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put our balance under counterparty-announced channel reserve value \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        }
@@ -1896,7 +1926,8 @@ fn test_channel_reserve_holding_cell_htlcs() {
        // now see if they go through on both sides
        let (route_21, our_payment_hash_21, our_payment_preimage_21, our_payment_secret_21) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_21);
        // but this will stuck in the holding cell
-       nodes[0].node.send_payment(&route_21, our_payment_hash_21, &Some(our_payment_secret_21), PaymentId(our_payment_hash_21.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route_21, our_payment_hash_21,
+               RecipientOnionFields::secret_only(our_payment_secret_21), PaymentId(our_payment_hash_21.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 0);
@@ -1904,7 +1935,9 @@ fn test_channel_reserve_holding_cell_htlcs() {
        // test with outbound holding cell amount > 0
        {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_22+1);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+                       ), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put our balance under counterparty-announced channel reserve value \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
                nodes[0].logger.assert_log_contains("lightning::ln::channelmanager", "Cannot send value that would put our balance under counterparty-announced channel reserve value", 2);
@@ -1912,7 +1945,8 @@ fn test_channel_reserve_holding_cell_htlcs() {
 
        let (route_22, our_payment_hash_22, our_payment_preimage_22, our_payment_secret_22) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_22);
        // this will also stuck in the holding cell
-       nodes[0].node.send_payment(&route_22, our_payment_hash_22, &Some(our_payment_secret_22), PaymentId(our_payment_hash_22.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route_22, our_payment_hash_22,
+               RecipientOnionFields::secret_only(our_payment_secret_22), PaymentId(our_payment_hash_22.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
        assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -1963,7 +1997,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
        let events = nodes[2].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
        match events[0] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(our_payment_hash_21, *payment_hash);
                        assert_eq!(recv_value_21, amount_msat);
                        assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
@@ -1979,7 +2013,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
                _ => panic!("Unexpected event"),
        }
        match events[1] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(our_payment_hash_22, *payment_hash);
                        assert_eq!(recv_value_22, amount_msat);
                        assert_eq!(nodes[2].node.get_our_node_id(), receiver_node_id.unwrap());
@@ -2052,7 +2086,8 @@ fn channel_reserve_in_flight_removes() {
        // Start routing the third HTLC (this is just used to get everyone in the right state).
        let (route, payment_hash_3, payment_preimage_3, payment_secret_3) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
        let send_1 = {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_3,
+                       RecipientOnionFields::secret_only(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -2126,7 +2161,8 @@ fn channel_reserve_in_flight_removes() {
        // to A to ensure that A doesn't count the almost-removed HTLC in update_add processing.
        let (route, payment_hash_4, payment_preimage_4, payment_secret_4) = get_route_and_payment_hash!(nodes[1], nodes[0], 10000);
        let send_2 = {
-               nodes[1].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
+               nodes[1].node.send_payment_with_route(&route, payment_hash_4,
+                       RecipientOnionFields::secret_only(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
                let mut events = nodes[1].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -2354,8 +2390,8 @@ fn channel_monitor_network_test() {
 }
 
 #[test]
-fn test_justice_tx() {
-       // Test justice txn built on revoked HTLC-Success tx, against both sides
+fn test_justice_tx_htlc_timeout() {
+       // Test justice txn built on revoked HTLC-Timeout tx, against both sides
        let mut alice_config = UserConfig::default();
        alice_config.channel_handshake_config.announced_channel = true;
        alice_config.channel_handshake_limits.force_announced_channel_preference = false;
@@ -2371,7 +2407,6 @@ fn test_justice_tx() {
        let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &user_cfgs);
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
-       *nodes[0].connect_style.borrow_mut() = ConnectStyle::FullBlockViaListen;
        // Create some new channels:
        let chan_5 = create_announced_chan_between_nodes(&nodes, 0, 1);
 
@@ -2395,7 +2430,6 @@ fn test_justice_tx() {
                        let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                        assert_eq!(node_txn.len(), 1); // ChannelMonitor: penalty tx
                        assert_eq!(node_txn[0].input.len(), 2); // We should claim the revoked output and the HTLC output
-
                        check_spends!(node_txn[0], revoked_local_txn[0]);
                        node_txn.swap_remove(0);
                }
@@ -2404,7 +2438,7 @@ fn test_justice_tx() {
                test_txn_broadcast(&nodes[1], &chan_5, Some(revoked_local_txn[0].clone()), HTLCType::NONE);
 
                mine_transaction(&nodes[0], &revoked_local_txn[0]);
-               connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+               connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
                // Verify broadcast of revoked HTLC-timeout
                let node_txn = test_txn_broadcast(&nodes[0], &chan_5, Some(revoked_local_txn[0].clone()), HTLCType::TIMEOUT);
                check_added_monitors!(nodes[0], 1);
@@ -2414,17 +2448,30 @@ fn test_justice_tx() {
                test_revoked_htlc_claim_txn_broadcast(&nodes[1], node_txn[1].clone(), revoked_local_txn[0].clone());
        }
        get_announce_close_broadcast_events(&nodes, 0, 1);
-
        assert_eq!(nodes[0].node.list_channels().len(), 0);
        assert_eq!(nodes[1].node.list_channels().len(), 0);
+}
 
-       // We test justice_tx build by A on B's revoked HTLC-Success tx
+#[test]
+fn test_justice_tx_htlc_success() {
+       // Test justice txn built on revoked HTLC-Success tx, against both sides
+       let mut alice_config = UserConfig::default();
+       alice_config.channel_handshake_config.announced_channel = true;
+       alice_config.channel_handshake_limits.force_announced_channel_preference = false;
+       alice_config.channel_handshake_config.our_to_self_delay = 6 * 24 * 5;
+       let mut bob_config = UserConfig::default();
+       bob_config.channel_handshake_config.announced_channel = true;
+       bob_config.channel_handshake_limits.force_announced_channel_preference = false;
+       bob_config.channel_handshake_config.our_to_self_delay = 6 * 24 * 3;
+       let user_cfgs = [Some(alice_config), Some(bob_config)];
+       let mut chanmon_cfgs = create_chanmon_cfgs(2);
+       chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
+       chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true;
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &user_cfgs);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        // Create some new channels:
        let chan_6 = create_announced_chan_between_nodes(&nodes, 0, 1);
-       {
-               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               node_txn.clear();
-       }
 
        // A pending HTLC which will be revoked:
        let payment_preimage_4 = route_payment(&nodes[0], &vec!(&nodes[1])[..], 3000000).0;
@@ -2602,8 +2649,7 @@ fn claim_htlc_outputs_single_tx() {
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-               assert_eq!(node_txn.len(), 7);
+               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcast();
 
                // Check the pair local commitment and HTLC-timeout broadcast due to HTLC expiration
                assert_eq!(node_txn[0].input.len(), 1);
@@ -2613,19 +2659,22 @@ fn claim_htlc_outputs_single_tx() {
                assert_eq!(witness_script.len(), OFFERED_HTLC_SCRIPT_WEIGHT); //Spending an offered htlc output
                check_spends!(node_txn[1], node_txn[0]);
 
-               // Justice transactions are indices 2-3-4
+               // Filter out any non justice transactions.
+               node_txn.retain(|tx| tx.input[0].previous_output.txid == revoked_local_txn[0].txid());
+               assert!(node_txn.len() > 3);
+
+               assert_eq!(node_txn[0].input.len(), 1);
+               assert_eq!(node_txn[1].input.len(), 1);
                assert_eq!(node_txn[2].input.len(), 1);
-               assert_eq!(node_txn[3].input.len(), 1);
-               assert_eq!(node_txn[4].input.len(), 1);
 
+               check_spends!(node_txn[0], revoked_local_txn[0]);
+               check_spends!(node_txn[1], revoked_local_txn[0]);
                check_spends!(node_txn[2], revoked_local_txn[0]);
-               check_spends!(node_txn[3], revoked_local_txn[0]);
-               check_spends!(node_txn[4], revoked_local_txn[0]);
 
                let mut witness_lens = BTreeSet::new();
+               witness_lens.insert(node_txn[0].input[0].witness.last().unwrap().len());
+               witness_lens.insert(node_txn[1].input[0].witness.last().unwrap().len());
                witness_lens.insert(node_txn[2].input[0].witness.last().unwrap().len());
-               witness_lens.insert(node_txn[3].input[0].witness.last().unwrap().len());
-               witness_lens.insert(node_txn[4].input[0].witness.last().unwrap().len());
                assert_eq!(witness_lens.len(), 3);
                assert_eq!(*witness_lens.iter().skip(0).next().unwrap(), 77); // revoked to_local
                assert_eq!(*witness_lens.iter().skip(1).next().unwrap(), OFFERED_HTLC_SCRIPT_WEIGHT); // revoked offered HTLC
@@ -2633,9 +2682,9 @@ fn claim_htlc_outputs_single_tx() {
 
                // Finally, mine the penalty transactions and check that we get an HTLC failure after
                // ANTI_REORG_DELAY confirmations.
+               mine_transaction(&nodes[1], &node_txn[0]);
+               mine_transaction(&nodes[1], &node_txn[1]);
                mine_transaction(&nodes[1], &node_txn[2]);
-               mine_transaction(&nodes[1], &node_txn[3]);
-               mine_transaction(&nodes[1], &node_txn[4]);
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                expect_payment_failed!(nodes[1], payment_hash_2, false);
        }
@@ -2716,7 +2765,7 @@ fn test_htlc_on_chain_success() {
        // Verify that B's ChannelManager is able to extract preimage from HTLC Success tx and pass it backward
        let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42};
        connect_block(&nodes[1], &Block { header, txdata: vec![commitment_tx[0].clone(), node_txn[0].clone(), node_txn[1].clone()]});
-       connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
        {
                let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
                assert_eq!(added_monitors.len(), 1);
@@ -2845,7 +2894,7 @@ fn test_htlc_on_chain_success() {
        assert_eq!(commitment_spend.input.len(), 2);
        assert_eq!(commitment_spend.input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
        assert_eq!(commitment_spend.input[1].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
-       assert_eq!(commitment_spend.lock_time.0, nodes[1].best_block_info().1 + 1);
+       assert_eq!(commitment_spend.lock_time.0, nodes[1].best_block_info().1);
        assert!(commitment_spend.output[0].script_pubkey.is_v0_p2wpkh()); // direct payment
        // We don't bother to check that B can claim the HTLC output on its commitment tx here as
        // we already checked the same situation with A.
@@ -2853,7 +2902,7 @@ fn test_htlc_on_chain_success() {
        // Verify that A's ChannelManager is able to extract preimage from preimage tx and generate PaymentSent
        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42};
        connect_block(&nodes[0], &Block { header, txdata: vec![node_a_commitment_tx[0].clone(), commitment_spend.clone()] });
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV + MIN_CLTV_EXPIRY_DELTA as u32 - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV + MIN_CLTV_EXPIRY_DELTA as u32); // Confirm blocks until the HTLC expires
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        let events = nodes[0].node.get_and_clear_pending_events();
@@ -2934,25 +2983,20 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
 
        // Broadcast timeout transaction by B on received output from C's commitment tx on B's chain
        // Verify that B's ChannelManager is able to detect that HTLC is timeout by its own tx and react backward in consequence
-       connect_blocks(&nodes[1], 200 - nodes[2].best_block_info().1);
        mine_transaction(&nodes[1], &commitment_tx[0]);
-       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       let timeout_tx;
-       {
-               let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               assert_eq!(node_txn.len(), 3); // 2 (local commitment tx + HTLC-timeout), 1 timeout tx
-
-               check_spends!(node_txn[2], commitment_tx[0]);
-               assert_eq!(node_txn[2].clone().input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-
-               check_spends!(node_txn[0], chan_2.3);
-               check_spends!(node_txn[1], node_txn[0]);
-               assert_eq!(node_txn[0].clone().input[0].witness.last().unwrap().len(), 71);
-               assert_eq!(node_txn[1].clone().input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
-
-               timeout_tx = node_txn[2].clone();
-               node_txn.clear();
-       }
+       check_closed_event(&nodes[1], 1, ClosureReason::CommitmentTxConfirmed, false);
+       connect_blocks(&nodes[1], 200 - nodes[2].best_block_info().1);
+       let timeout_tx = {
+               let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
+               if nodes[1].connect_style.borrow().skips_blocks() {
+                       assert_eq!(txn.len(), 1);
+               } else {
+                       assert_eq!(txn.len(), 3); // Two extra fee bumps for timeout transaction
+               }
+               txn.iter().for_each(|tx| check_spends!(tx, commitment_tx[0]));
+               assert_eq!(txn[0].clone().input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               txn.remove(0)
+       };
 
        mine_transaction(&nodes[1], &timeout_tx);
        check_added_monitors!(nodes[1], 1);
@@ -2980,7 +3024,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
        check_spends!(commitment_tx[0], chan_1.3);
 
        mine_transaction(&nodes[0], &commitment_tx[0]);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV + MIN_CLTV_EXPIRY_DELTA as u32 - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV + MIN_CLTV_EXPIRY_DELTA as u32); // Confirm blocks until the HTLC expires
 
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
@@ -3141,7 +3185,8 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        // Add a fourth HTLC, this one will get sequestered away in nodes[1]'s holding cell waiting
        // on nodes[2]'s RAA.
        let (route, fourth_payment_hash, _, fourth_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 1000000);
-       nodes[1].node.send_payment(&route, fourth_payment_hash, &Some(fourth_payment_secret), PaymentId(fourth_payment_hash.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, fourth_payment_hash,
+               RecipientOnionFields::secret_only(fourth_payment_secret), PaymentId(fourth_payment_hash.0)).unwrap();
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
        check_added_monitors!(nodes[1], 0);
@@ -3320,7 +3365,8 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        // Alice -> Bob: Route a payment but without Bob sending revoke_and_ack.
        {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 50_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                       PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let payment_event = {
@@ -3335,7 +3381,8 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        // Alice -> Bob: Route another payment but now Alice waits for Bob's earlier revoke_and_ack.
        let (route, failed_payment_hash, _, failed_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 50_000);
        {
-               nodes[0].node.send_payment(&route, failed_payment_hash, &Some(failed_payment_secret), PaymentId(failed_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, failed_payment_hash,
+                       RecipientOnionFields::secret_only(failed_payment_secret), PaymentId(failed_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
 
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -3348,7 +3395,8 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
                let secp_ctx = Secp256k1::new();
                let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
                let current_height = nodes[1].node.best_block.read().unwrap().height() + 1;
-               let (onion_payloads, _amount_msat, cltv_expiry) = onion_utils::build_onion_payloads(&route.paths[0], 50_000, &Some(payment_secret), current_height, &None).unwrap();
+               let (onion_payloads, _amount_msat, cltv_expiry) = onion_utils::build_onion_payloads(
+                       &route.paths[0], 50_000, RecipientOnionFields::secret_only(payment_secret), current_height, &None).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
                let onion_routing_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
 
@@ -3413,7 +3461,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(node_txn.len(), 3);
-       assert_eq!(node_txn[0], node_txn[1]);
+       assert_eq!(node_txn[0].txid(), node_txn[1].txid());
 
        let mut header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[1].clone()]});
@@ -3439,7 +3487,8 @@ fn test_force_close_fail_back() {
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
 
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -3663,7 +3712,8 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1_000_000);
 
        let payment_event = {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -3761,7 +3811,7 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let events_2 = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(events_2.len(), 1);
        match events_2[0] {
-               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, via_user_channel_id: _ } => {
+               Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id, .. } => {
                        assert_eq!(payment_hash_1, *payment_hash);
                        assert_eq!(amount_msat, 1_000_000);
                        assert_eq!(receiver_node_id.unwrap(), nodes[1].node.get_our_node_id());
@@ -3933,7 +3983,8 @@ fn test_drop_messages_peer_disconnect_dual_htlc() {
 
        // Now try to send a second payment which will fail to send
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
@@ -4085,8 +4136,11 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) {
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
                let payment_id = PaymentId([42; 32]);
-               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &route).unwrap();
-               nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), payment_id, &route).unwrap();
+               nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), 200_000, cur_height, payment_id,
+                       &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -4150,16 +4204,16 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
 
        // Route a first payment to get the 1 -> 2 channel in awaiting_raa...
        let (route, first_payment_hash, _, first_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
-       {
-               nodes[1].node.send_payment(&route, first_payment_hash, &Some(first_payment_secret), PaymentId(first_payment_hash.0)).unwrap();
-       }
+       nodes[1].node.send_payment_with_route(&route, first_payment_hash,
+               RecipientOnionFields::secret_only(first_payment_secret), PaymentId(first_payment_hash.0)).unwrap();
        assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
        check_added_monitors!(nodes[1], 1);
 
        // Now attempt to route a second payment, which should be placed in the holding cell
        let sending_node = if forwarded_htlc { &nodes[0] } else { &nodes[1] };
        let (route, second_payment_hash, _, second_payment_secret) = get_route_and_payment_hash!(sending_node, nodes[2], 100000);
-       sending_node.node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
+       sending_node.node.send_payment_with_route(&route, second_payment_hash,
+               RecipientOnionFields::secret_only(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
        if forwarded_htlc {
                check_added_monitors!(nodes[0], 1);
                let payment_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
@@ -4392,7 +4446,7 @@ fn test_static_spendable_outputs_timeout_tx() {
                MessageSendEvent::BroadcastChannelUpdate { .. } => {},
                _ => panic!("Unexpected event"),
        }
-       connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
 
        // Check B's monitor was able to send back output descriptor event for timeout tx on A's commitment tx
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
@@ -4470,7 +4524,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() {
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
 
        let revoked_htlc_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(revoked_htlc_txn.len(), 1);
@@ -4702,7 +4756,7 @@ fn test_onchain_to_onchain_claim() {
        check_spends!(b_txn[0], commitment_tx[0]);
        assert_eq!(b_txn[0].input[0].witness.clone().last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
        assert!(b_txn[0].output[0].script_pubkey.is_v0_p2wpkh()); // direct payment
-       assert_eq!(b_txn[0].lock_time.0, nodes[1].best_block_info().1 + 1); // Success tx
+       assert_eq!(b_txn[0].lock_time.0, nodes[1].best_block_info().1); // Success tx
 
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
@@ -4753,7 +4807,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[1], TEST_FINAL_CLTV - 40 + MIN_CLTV_EXPIRY_DELTA as u32 - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV - 40 + MIN_CLTV_EXPIRY_DELTA as u32); // Confirm blocks until the HTLC expires
 
        let htlc_timeout_tx;
        { // Extract one of the two HTLC-Timeout transaction
@@ -4851,15 +4905,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
 
        nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], &updates.commitment_signed, false);
-
-       let events = nodes[0].node.get_and_clear_pending_events();
-       match events[0] {
-               Event::PaymentSent { ref payment_preimage, ref payment_hash, .. } => {
-                       assert_eq!(*payment_preimage, our_payment_preimage);
-                       assert_eq!(*payment_hash, duplicate_payment_hash);
-               }
-               _ => panic!("Unexpected event"),
-       }
+       expect_payment_sent(&nodes[0], our_payment_preimage, None, true);
 }
 
 #[test]
@@ -5241,7 +5287,7 @@ fn test_dynamic_spendable_outputs_local_htlc_timeout_tx() {
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
 
        let htlc_timeout = {
                let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -5324,7 +5370,7 @@ fn test_key_derivation_params() {
 
        // Timeout HTLC on A's chain and so it can generate a HTLC-Timeout tx
        mine_transaction(&nodes[0], &local_txn_1[0]);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
@@ -5433,7 +5479,8 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1);
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], if use_dust { 50000 } else { 3000000 });
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let _as_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -5672,7 +5719,8 @@ fn test_fail_holding_cell_htlc_upon_free() {
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send);
 
        // Send a payment which passes reserve checks but gets stuck in the holding cell.
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], nodes[1], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, max_can_send);
 
@@ -5701,7 +5749,7 @@ fn test_fail_holding_cell_htlc_upon_free() {
                        assert_eq!(PaymentId(our_payment_hash.0), *payment_id.as_ref().unwrap());
                        assert_eq!(our_payment_hash.clone(), *payment_hash);
                        assert_eq!(*payment_failed_permanently, false);
-                       assert_eq!(*short_channel_id, Some(route.paths[0][0].short_channel_id));
+                       assert_eq!(*short_channel_id, Some(route.paths[0].hops[0].short_channel_id));
                },
                _ => panic!("Unexpected event"),
        }
@@ -5756,11 +5804,13 @@ fn test_free_and_fail_holding_cell_htlcs() {
        let (route_2, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], amt_2);
 
        // Send 2 payments which pass reserve checks but get stuck in the holding cell.
-       nodes[0].node.send_payment(&route_1, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route_1, payment_hash_1,
+               RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], nodes[1], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, amt_1);
        let payment_id_2 = PaymentId(nodes[0].keys_manager.get_secure_random_bytes());
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), payment_id_2).unwrap();
+       nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), payment_id_2).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], nodes[1], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, amt_1 + amt_2);
 
@@ -5790,7 +5840,7 @@ fn test_free_and_fail_holding_cell_htlcs() {
                        assert_eq!(payment_id_2, *payment_id.as_ref().unwrap());
                        assert_eq!(payment_hash_2.clone(), *payment_hash);
                        assert_eq!(*payment_failed_permanently, false);
-                       assert_eq!(*short_channel_id, Some(route_2.paths[0][0].short_channel_id));
+                       assert_eq!(*short_channel_id, Some(route_2.paths[0].hops[0].short_channel_id));
                },
                _ => panic!("Unexpected event"),
        }
@@ -5886,7 +5936,8 @@ fn test_fail_holding_cell_htlc_upon_free_multihop() {
        let max_can_send = 5000000 - channel_reserve - 2*commit_tx_fee_msat(feerate, 1 + 1, opt_anchors) - total_routing_fee_msat;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], max_can_send);
        let payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -5986,9 +6037,11 @@ fn test_update_add_htlc_bolt2_sender_value_below_minimum_msat() {
        let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000);
 
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       route.paths[0][0].fee_msat = 100;
+       route.paths[0].hops[0].fee_msat = 100;
 
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot send less than their minimum HTLC value \(\d+\)").unwrap().is_match(err)));
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        nodes[0].logger.assert_log_contains("lightning::ln::channelmanager", "Cannot send less than their minimum HTLC value", 1);
@@ -6004,8 +6057,10 @@ fn test_update_add_htlc_bolt2_sender_zero_value_msat() {
        let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000);
 
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       route.paths[0][0].fee_msat = 0;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       route.paths[0].hops[0].fee_msat = 0;
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)),
+               true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send 0-msat HTLC"));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6022,7 +6077,8 @@ fn test_update_add_htlc_bolt2_receiver_zero_value_msat() {
        let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = 0;
@@ -6047,8 +6103,10 @@ fn test_update_add_htlc_bolt2_sender_cltv_expiry_too_high() {
        let payment_params = PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), 0)
                .with_features(nodes[1].node.invoice_features());
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], payment_params, 100000000, 0);
-       route.paths[0].last_mut().unwrap().cltv_expiry_delta = 500000001;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::InvalidRoute { ref err },
+       route.paths[0].hops.last_mut().unwrap().cltv_expiry_delta = 500000001;
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::InvalidRoute { ref err },
                assert_eq!(err, &"Channel CLTV overflowed?"));
 }
 
@@ -6068,7 +6126,8 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
        for i in 0..max_accepted_htlcs {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
                let payment_event = {
-                       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+                       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                        check_added_monitors!(nodes[0], 1);
 
                        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -6088,7 +6147,9 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
                expect_payment_claimable!(nodes[1], our_payment_hash, our_payment_secret, 100000);
        }
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6111,8 +6172,10 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_value_in_flight() {
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_in_flight);
        // Manually create a route over our max in flight (which our router normally automatically
        // limits us to.
-       route.paths[0][0].fee_msat =  max_in_flight + 1;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
+       route.paths[0].hops[0].fee_msat =  max_in_flight + 1;
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot send value that would put us over the max HTLC value in flight our peer will accept \(\d+\)").unwrap().is_match(err)));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6139,7 +6202,8 @@ fn test_update_add_htlc_bolt2_receiver_check_amount_received_more_than_min() {
        }
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], htlc_minimum_msat);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = htlc_minimum_msat-1;
@@ -6169,7 +6233,8 @@ fn test_update_add_htlc_bolt2_receiver_sender_can_afford_amount_sent() {
 
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee_outbound;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 
@@ -6200,7 +6265,8 @@ fn test_update_add_htlc_bolt2_receiver_check_max_htlc_limit() {
        let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
        let cur_height = nodes[0].node.best_block.read().unwrap().height() + 1;
        let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::signing_only(), &route.paths[0], &session_priv).unwrap();
-       let (onion_payloads, _htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 3999999, &Some(our_payment_secret), cur_height, &None).unwrap();
+       let (onion_payloads, _htlc_msat, htlc_cltv) = onion_utils::build_onion_payloads(
+               &route.paths[0], 3999999, RecipientOnionFields::secret_only(our_payment_secret), cur_height, &None).unwrap();
        let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &our_payment_hash);
 
        let mut msg = msgs::UpdateAddHTLC {
@@ -6212,11 +6278,11 @@ fn test_update_add_htlc_bolt2_receiver_check_max_htlc_limit() {
                onion_routing_packet: onion_packet.clone(),
        };
 
-       for i in 0..super::channel::OUR_MAX_HTLCS {
+       for i in 0..50 {
                msg.htlc_id = i as u64;
                nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &msg);
        }
-       msg.htlc_id = (super::channel::OUR_MAX_HTLCS) as u64;
+       msg.htlc_id = (50) as u64;
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &msg);
 
        assert!(nodes[1].node.list_channels().is_empty());
@@ -6236,7 +6302,8 @@ fn test_update_add_htlc_bolt2_receiver_check_max_in_flight_msat() {
        let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = get_channel_value_stat!(nodes[1], nodes[0], chan.2).counterparty_max_htlc_value_in_flight_msat + 1;
@@ -6259,7 +6326,8 @@ fn test_update_add_htlc_bolt2_receiver_check_cltv_expiry() {
 
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000);
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].cltv_expiry = 500000000;
@@ -6284,7 +6352,8 @@ fn test_update_add_htlc_bolt2_receiver_check_repeated_id_ignore() {
 
        create_announced_chan_between_nodes(&nodes, 0, 1);
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -6329,7 +6398,8 @@ fn test_update_fulfill_htlc_bolt2_update_fulfill_htlc_before_commitment() {
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1);
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
 
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6361,7 +6431,8 @@ fn test_update_fulfill_htlc_bolt2_update_fail_htlc_before_commitment() {
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -6392,7 +6463,8 @@ fn test_update_fulfill_htlc_bolt2_update_fail_malformed_htlc_before_commitment()
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -6509,7 +6581,8 @@ fn test_update_fulfill_htlc_bolt2_missing_badonion_bit_for_malformed_htlc_messag
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6560,7 +6633,8 @@ fn test_update_fulfill_htlc_bolt2_after_malformed_htlc_message_must_forward_upda
 
        //First hop
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -6634,7 +6708,8 @@ fn test_channel_failed_after_message_with_badonion_node_perm_bits_set() {
 
        // First hop
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                SendEvent::from_node(&nodes[0])
        };
@@ -6852,7 +6927,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
                check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
 
-               connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+               connect_blocks(&nodes[0], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
                timeout_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().drain(..)
                        .filter(|tx| tx.input[0].previous_output.txid == bs_commitment_tx[0].txid()).collect();
                check_spends!(timeout_tx[0], bs_commitment_tx[0]);
@@ -6863,7 +6938,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
                if !revoked {
                        assert_eq!(timeout_tx[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
                } else {
-                       assert_eq!(timeout_tx[0].lock_time.0, 12);
+                       assert_eq!(timeout_tx[0].lock_time.0, 11);
                }
                // We fail non-dust-HTLC 2 by broadcast of local timeout/revocation-claim tx
                mine_transaction(&nodes[0], &timeout_tx[0]);
@@ -6972,7 +7047,8 @@ fn test_check_htlc_underpaying() {
        let route = get_route(&nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(), None, 10_000, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
        let (_, our_payment_hash, _) = get_payment_preimage_hash!(nodes[0]);
        let our_payment_secret = nodes[1].node.create_inbound_payment_for_hash(our_payment_hash, Some(100_000), 7200, None).unwrap();
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7032,8 +7108,9 @@ fn test_announce_disable_channels() {
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 
-       nodes[0].node.timer_tick_occurred(); // Enabled -> DisabledStaged
-       nodes[0].node.timer_tick_occurred(); // DisabledStaged -> Disabled
+       for _ in 0..DISABLE_GOSSIP_TICKS + 1 {
+               nodes[0].node.timer_tick_occurred();
+       }
        let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(msg_events.len(), 3);
        let mut chans_disabled = HashMap::new();
@@ -7073,7 +7150,9 @@ fn test_announce_disable_channels() {
        nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[2]);
        handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 
-       nodes[0].node.timer_tick_occurred();
+       for _ in 0..ENABLE_GOSSIP_TICKS {
+               nodes[0].node.timer_tick_occurred();
+       }
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        nodes[0].node.timer_tick_occurred();
        let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7239,19 +7318,23 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[1], 49); // Confirm blocks until the HTLC expires (note CLTV was explicitly 50 above)
+       connect_blocks(&nodes[1], 50); // Confirm blocks until the HTLC expires (note CLTV was explicitly 50 above)
 
-       let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(revoked_htlc_txn.len(), 2);
+       let revoked_htlc_txn = {
+               let txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
 
-       assert_eq!(revoked_htlc_txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-       assert_eq!(revoked_htlc_txn[0].input.len(), 1);
-       check_spends!(revoked_htlc_txn[0], revoked_local_txn[0]);
+               assert_eq!(txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               assert_eq!(txn[0].input.len(), 1);
+               check_spends!(txn[0], revoked_local_txn[0]);
 
-       assert_eq!(revoked_htlc_txn[1].input.len(), 1);
-       assert_eq!(revoked_htlc_txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
-       assert_eq!(revoked_htlc_txn[1].output.len(), 1);
-       check_spends!(revoked_htlc_txn[1], revoked_local_txn[0]);
+               assert_eq!(txn[1].input.len(), 1);
+               assert_eq!(txn[1].input[0].witness.last().unwrap().len(), OFFERED_HTLC_SCRIPT_WEIGHT);
+               assert_eq!(txn[1].output.len(), 1);
+               check_spends!(txn[1], revoked_local_txn[0]);
+
+               txn
+       };
 
        // Broadcast set of revoked txn on A
        let hash_128 = connect_blocks(&nodes[0], 40);
@@ -7381,7 +7464,7 @@ fn test_bump_penalty_txn_on_remote_commitment() {
        expect_payment_claimed!(nodes[1], payment_hash, 3_000_000);
        mine_transaction(&nodes[1], &remote_txn[0]);
        check_added_monitors!(nodes[1], 2);
-       connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
 
        // One or more claim tx should have been broadcast, check it
        let timeout;
@@ -7427,7 +7510,7 @@ fn test_bump_penalty_txn_on_remote_commitment() {
        assert_ne!(feerate_preimage, 0);
 
        // After exhaustion of height timer, new bumped claim txn should have been broadcast, check it
-       connect_blocks(&nodes[1], 15);
+       connect_blocks(&nodes[1], 1);
        {
                let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                assert_eq!(node_txn.len(), 1);
@@ -7494,7 +7577,13 @@ fn test_counterparty_raa_skip_no_crash() {
        }
 
        nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(),
-               &msgs::RevokeAndACK { channel_id, per_commitment_secret, next_per_commitment_point });
+               &msgs::RevokeAndACK {
+                       channel_id,
+                       per_commitment_secret,
+                       next_per_commitment_point,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
+               });
        assert_eq!(check_closed_broadcast!(nodes[1], true).unwrap().data, "Received an unexpected revoke_and_ack");
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "Received an unexpected revoke_and_ack".to_string() });
@@ -7582,9 +7671,10 @@ fn test_pending_claimed_htlc_no_balance_underflow() {
        // almost-claimed HTLC as available balance.
        let (mut route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 10_000);
        route.payment_params = None; // This is all wrong, but unnecessary
-       route.paths[0][0].pubkey = nodes[0].node.get_our_node_id();
+       route.paths[0].hops[0].pubkey = nodes[0].node.get_our_node_id();
        let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
-       nodes[1].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       nodes[1].node.send_payment_with_route(&route, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 
        assert_eq!(nodes[1].node.list_channels()[0].balance_msat, 1_000_000);
 }
@@ -7953,25 +8043,27 @@ fn test_onion_value_mpp_set_calculation() {
        let sample_path = route.paths.pop().unwrap();
 
        let mut path_1 = sample_path.clone();
-       path_1[0].pubkey = nodes[1].node.get_our_node_id();
-       path_1[0].short_channel_id = chan_1_id;
-       path_1[1].pubkey = nodes[3].node.get_our_node_id();
-       path_1[1].short_channel_id = chan_3_id;
-       path_1[1].fee_msat = 100_000;
+       path_1.hops[0].pubkey = nodes[1].node.get_our_node_id();
+       path_1.hops[0].short_channel_id = chan_1_id;
+       path_1.hops[1].pubkey = nodes[3].node.get_our_node_id();
+       path_1.hops[1].short_channel_id = chan_3_id;
+       path_1.hops[1].fee_msat = 100_000;
        route.paths.push(path_1);
 
        let mut path_2 = sample_path.clone();
-       path_2[0].pubkey = nodes[2].node.get_our_node_id();
-       path_2[0].short_channel_id = chan_2_id;
-       path_2[1].pubkey = nodes[3].node.get_our_node_id();
-       path_2[1].short_channel_id = chan_4_id;
-       path_2[1].fee_msat = 1_000;
+       path_2.hops[0].pubkey = nodes[2].node.get_our_node_id();
+       path_2.hops[0].short_channel_id = chan_2_id;
+       path_2.hops[1].pubkey = nodes[3].node.get_our_node_id();
+       path_2.hops[1].short_channel_id = chan_4_id;
+       path_2.hops[1].fee_msat = 1_000;
        route.paths.push(path_2);
 
        // Send payment
        let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
-       let onion_session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(our_payment_secret), payment_id, &route).unwrap();
-       nodes[0].node.test_send_payment_internal(&route, our_payment_hash, &Some(our_payment_secret), None, payment_id, Some(total_msat), onion_session_privs).unwrap();
+       let onion_session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), payment_id, &route).unwrap();
+       nodes[0].node.test_send_payment_internal(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), None, payment_id, Some(total_msat), onion_session_privs).unwrap();
        check_added_monitors!(nodes[0], expected_paths.len());
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7990,7 +8082,8 @@ fn test_onion_value_mpp_set_calculation() {
                        let height = nodes[0].best_block_info().1;
                        let session_priv = SecretKey::from_slice(&session_priv).unwrap();
                        let mut onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
-                       let (mut onion_payloads, _, _) = onion_utils::build_onion_payloads(&route.paths[0], 100_000, &Some(our_payment_secret), height + 1, &None).unwrap();
+                       let (mut onion_payloads, _, _) = onion_utils::build_onion_payloads(&route.paths[0], 100_000,
+                               RecipientOnionFields::secret_only(our_payment_secret), height + 1, &None).unwrap();
                        // Edit amt_to_forward to simulate the sender having set
                        // the final amount and the routing node taking less fee
                        onion_payloads[1].amt_to_forward = 99_000;
@@ -8059,18 +8152,20 @@ fn do_test_overshoot_mpp(msat_amounts: &[u64], total_msat: u64) {
        for i in 0..routing_node_count {
                let routing_node = 2 + i;
                let mut path = sample_path.clone();
-               path[0].pubkey = nodes[routing_node].node.get_our_node_id();
-               path[0].short_channel_id = src_chan_ids[i];
-               path[1].pubkey = nodes[dst_idx].node.get_our_node_id();
-               path[1].short_channel_id = dst_chan_ids[i];
-               path[1].fee_msat = msat_amounts[i];
+               path.hops[0].pubkey = nodes[routing_node].node.get_our_node_id();
+               path.hops[0].short_channel_id = src_chan_ids[i];
+               path.hops[1].pubkey = nodes[dst_idx].node.get_our_node_id();
+               path.hops[1].short_channel_id = dst_chan_ids[i];
+               path.hops[1].fee_msat = msat_amounts[i];
                route.paths.push(path);
        }
 
        // Send payment with manually set total_msat
        let payment_id = PaymentId(nodes[src_idx].keys_manager.backing.get_secure_random_bytes());
-       let onion_session_privs = nodes[src_idx].node.test_add_new_pending_payment(our_payment_hash, Some(our_payment_secret), payment_id, &route).unwrap();
-       nodes[src_idx].node.test_send_payment_internal(&route, our_payment_hash, &Some(our_payment_secret), None, payment_id, Some(total_msat), onion_session_privs).unwrap();
+       let onion_session_privs = nodes[src_idx].node.test_add_new_pending_payment(our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), payment_id, &route).unwrap();
+       nodes[src_idx].node.test_send_payment_internal(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), None, payment_id, Some(total_msat), onion_session_privs).unwrap();
        check_added_monitors!(nodes[src_idx], expected_paths.len());
 
        let mut events = nodes[src_idx].node.get_and_clear_pending_msg_events();
@@ -8110,12 +8205,12 @@ fn test_simple_mpp() {
        let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
        let path = route.paths[0].clone();
        route.paths.push(path);
-       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-       route.paths[0][0].short_channel_id = chan_1_id;
-       route.paths[0][1].short_channel_id = chan_3_id;
-       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-       route.paths[1][0].short_channel_id = chan_2_id;
-       route.paths[1][1].short_channel_id = chan_4_id;
+       route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0].hops[0].short_channel_id = chan_1_id;
+       route.paths[0].hops[1].short_channel_id = chan_3_id;
+       route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1].hops[0].short_channel_id = chan_2_id;
+       route.paths[1].hops[1].short_channel_id = chan_4_id;
        send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
        claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
 }
@@ -8133,7 +8228,8 @@ fn test_preimage_storage() {
        {
                let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(100_000), 7200, None).unwrap();
                let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                let mut payment_event = SendEvent::from_event(events.pop().unwrap());
@@ -8203,7 +8299,8 @@ fn test_secret_timeout() {
 
        {
                let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(our_payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                let mut payment_event = SendEvent::from_event(events.pop().unwrap());
@@ -8273,17 +8370,20 @@ fn test_bad_secret_hash() {
        let expected_error_data = [0, 0, 0, 0, 0, 1, 0x86, 0xa0, 0, 0, 0, CHAN_CONFIRM_DEPTH as u8];
 
        // Send a payment with the right payment hash but the wrong payment secret
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(random_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(random_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(our_payment_hash);
        expect_payment_failed!(nodes[0], our_payment_hash, true, expected_error_code, expected_error_data);
 
        // Send a payment with a random payment hash, but the right payment secret
-       nodes[0].node.send_payment(&route, random_payment_hash, &Some(our_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, random_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(random_payment_hash);
        expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data);
 
        // Send a payment with a random payment hash and random payment secret
-       nodes[0].node.send_payment(&route, random_payment_hash, &Some(random_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, random_payment_hash,
+               RecipientOnionFields::secret_only(random_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(random_payment_hash);
        expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data);
 }
@@ -8334,7 +8434,7 @@ fn test_update_err_monitor_lockdown() {
        let block = Block { header, txdata: vec![] };
        // Make the tx_broadcaster aware of enough blocks that it doesn't think we're violating
        // transaction lock time requirements here.
-       chanmon_cfgs[0].tx_broadcaster.blocks.lock().unwrap().resize(200, (block.clone(), 0));
+       chanmon_cfgs[0].tx_broadcaster.blocks.lock().unwrap().resize(200, (block.clone(), 200));
        watchtower.chain_monitor.block_connected(&block, 200);
 
        // Try to update ChannelMonitor
@@ -8386,6 +8486,9 @@ fn test_concurrent_monitor_claim() {
        let chain_source = test_utils::TestChainSource::new(Network::Testnet);
        let logger = test_utils::TestLogger::with_id(format!("node {}", "Alice"));
        let persister = test_utils::TestPersister::new();
+       let alice_broadcaster = test_utils::TestBroadcaster::with_blocks(
+               Arc::new(Mutex::new(nodes[0].blocks.lock().unwrap().clone())),
+       );
        let watchtower_alice = {
                let new_monitor = {
                        let monitor = nodes[0].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap();
@@ -8394,28 +8497,30 @@ fn test_concurrent_monitor_claim() {
                        assert!(new_monitor == *monitor);
                        new_monitor
                };
-               let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
+               let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &alice_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(watchtower.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
                watchtower
        };
        let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
        let block = Block { header, txdata: vec![] };
-       // Make the tx_broadcaster aware of enough blocks that it doesn't think we're violating
-       // transaction lock time requirements here.
-       chanmon_cfgs[0].tx_broadcaster.blocks.lock().unwrap().resize((CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS) as usize, (block.clone(), 0));
-       watchtower_alice.chain_monitor.block_connected(&block, CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
+       // Make Alice aware of enough blocks that it doesn't think we're violating transaction lock time
+       // requirements here.
+       const HTLC_TIMEOUT_BROADCAST: u32 = CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS;
+       alice_broadcaster.blocks.lock().unwrap().resize((HTLC_TIMEOUT_BROADCAST) as usize, (block.clone(), HTLC_TIMEOUT_BROADCAST));
+       watchtower_alice.chain_monitor.block_connected(&block, HTLC_TIMEOUT_BROADCAST);
 
        // Watchtower Alice should have broadcast a commitment/HTLC-timeout
-       {
-               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
+       let alice_state = {
+               let mut txn = alice_broadcaster.txn_broadcast();
                assert_eq!(txn.len(), 2);
-               txn.clear();
-       }
+               txn.remove(0)
+       };
 
        // Copy ChainMonitor to simulate watchtower Bob and make it receive a commitment update first.
        let chain_source = test_utils::TestChainSource::new(Network::Testnet);
        let logger = test_utils::TestLogger::with_id(format!("node {}", "Bob"));
        let persister = test_utils::TestPersister::new();
+       let bob_broadcaster = test_utils::TestBroadcaster::with_blocks(Arc::clone(&alice_broadcaster.blocks));
        let watchtower_bob = {
                let new_monitor = {
                        let monitor = nodes[0].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap();
@@ -8424,18 +8529,17 @@ fn test_concurrent_monitor_claim() {
                        assert!(new_monitor == *monitor);
                        new_monitor
                };
-               let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
+               let watchtower = test_utils::TestChainMonitor::new(Some(&chain_source), &bob_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
                assert_eq!(watchtower.watch_channel(outpoint, new_monitor), ChannelMonitorUpdateStatus::Completed);
                watchtower
        };
        let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
-       watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, CHAN_CONFIRM_DEPTH + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
+       watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, HTLC_TIMEOUT_BROADCAST - 1);
 
        // Route another payment to generate another update with still previous HTLC pending
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 3000000);
-       {
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
-       }
+       nodes[1].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
@@ -8457,24 +8561,31 @@ fn test_concurrent_monitor_claim() {
 
        //// Provide one more block to watchtower Bob, expect broadcast of commitment and HTLC-Timeout
        let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
-       watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, CHAN_CONFIRM_DEPTH + 1 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
+       watchtower_bob.chain_monitor.block_connected(&Block { header, txdata: vec![] }, HTLC_TIMEOUT_BROADCAST);
 
        // Watchtower Bob should have broadcast a commitment/HTLC-timeout
        let bob_state_y;
        {
-               let mut txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
+               let mut txn = bob_broadcaster.txn_broadcast();
                assert_eq!(txn.len(), 2);
-               bob_state_y = txn[0].clone();
-               txn.clear();
+               bob_state_y = txn.remove(0);
        };
 
        // We confirm Bob's state Y on Alice, she should broadcast a HTLC-timeout
        let header = BlockHeader { version: 0x20000000, prev_blockhash: BlockHash::all_zeros(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
-       watchtower_alice.chain_monitor.block_connected(&Block { header, txdata: vec![bob_state_y.clone()] }, CHAN_CONFIRM_DEPTH + 2 + TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS);
+       let height = HTLC_TIMEOUT_BROADCAST + 1;
+       connect_blocks(&nodes[0], height - nodes[0].best_block_info().1);
+       check_closed_broadcast(&nodes[0], 1, true);
+       check_closed_event(&nodes[0], 1, ClosureReason::CommitmentTxConfirmed, false);
+       watchtower_alice.chain_monitor.block_connected(&Block { header, txdata: vec![bob_state_y.clone()] }, height);
+       check_added_monitors(&nodes[0], 1);
        {
-               let htlc_txn = chanmon_cfgs[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               assert_eq!(htlc_txn.len(), 1);
+               let htlc_txn = alice_broadcaster.txn_broadcast();
+               assert_eq!(htlc_txn.len(), 2);
                check_spends!(htlc_txn[0], bob_state_y);
+               // Alice doesn't clean up the old HTLC claim since it hasn't seen a conflicting spend for
+               // it. However, she should, because it now has an invalid parent.
+               check_spends!(htlc_txn[1], alice_state);
        }
 }
 
@@ -8549,7 +8660,7 @@ fn test_htlc_no_detection() {
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1);
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV);
 
        let htlc_timeout = {
                let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -8854,6 +8965,8 @@ fn test_duplicate_chan_id() {
                assert_eq!(added_monitors[0].0, funding_output);
                added_monitors.clear();
        }
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
+
        let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
 
        let funding_outpoint = crate::chain::transaction::OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index };
@@ -8929,6 +9042,7 @@ fn test_duplicate_chan_id() {
                assert_eq!(added_monitors[0].0, funding_output);
                added_monitors.clear();
        }
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
 
        let events_4 = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events_4.len(), 0);
@@ -9043,9 +9157,11 @@ fn test_invalid_funding_tx() {
        nodes[0].node.funding_transaction_generated_unchecked(&temporary_channel_id, &nodes[1].node.get_our_node_id(), tx.clone(), 0).unwrap();
        nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
        check_added_monitors!(nodes[1], 1);
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
        check_added_monitors!(nodes[0], 1);
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
 
        let events_1 = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events_1.len(), 0);
@@ -9142,7 +9258,7 @@ fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_t
                // We should broadcast an HTLC transaction spending our funding transaction first
                let spending_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                assert_eq!(spending_txn.len(), 2);
-               assert_eq!(spending_txn[0], node_txn[0]);
+               assert_eq!(spending_txn[0].txid(), node_txn[0].txid());
                check_spends!(spending_txn[1], node_txn[0]);
                // We should also generate a SpendableOutputs event with the to_self output (as its
                // timelock is up).
@@ -9190,7 +9306,8 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(&nodes[1]);
 
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -9203,7 +9320,8 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
 
        {
                // Note that we use a different PaymentId here to allow us to duplicativly pay
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_secret.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_secret.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -9296,7 +9414,7 @@ fn test_inconsistent_mpp_params() {
        assert_eq!(route.paths.len(), 2);
        route.paths.sort_by(|path_a, _| {
                // Sort the path so that the path through nodes[1] comes first
-               if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
+               if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
                        core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
        });
 
@@ -9310,9 +9428,12 @@ fn test_inconsistent_mpp_params() {
                // ultimately have, just not right away.
                let mut dup_route = route.clone();
                dup_route.paths.push(route.paths[1].clone());
-               nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(our_payment_secret), payment_id, &dup_route).unwrap()
+               nodes[0].node.test_add_new_pending_payment(our_payment_hash,
+                       RecipientOnionFields::secret_only(our_payment_secret), payment_id, &dup_route).unwrap()
        };
-       nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
+       nodes[0].node.test_send_payment_along_path(&route.paths[0], &our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), 15_000_000, cur_height, payment_id,
+               &None, session_privs[0]).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        {
@@ -9322,7 +9443,8 @@ fn test_inconsistent_mpp_params() {
        }
        assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
 
-       nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash, &Some(our_payment_secret), 14_000_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
+       nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), 14_000_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        {
@@ -9368,7 +9490,9 @@ fn test_inconsistent_mpp_params() {
 
        expect_payment_failed_conditions(&nodes[0], our_payment_hash, true, PaymentFailedConditions::new().mpp_parts_remain());
 
-       nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None, session_privs[2]).unwrap();
+       nodes[0].node.test_send_payment_along_path(&route.paths[1], &our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), 15_000_000, cur_height, payment_id,
+               &None, session_privs[2]).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9376,26 +9500,7 @@ fn test_inconsistent_mpp_params() {
        pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, our_payment_hash, Some(our_payment_secret), events.pop().unwrap(), true, None);
 
        do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, our_payment_preimage);
-       let events = nodes[0].node.get_and_clear_pending_events();
-       assert_eq!(events.len(), 3);
-       match events[0] {
-               Event::PaymentSent { payment_hash, .. } => { // The payment was abandoned earlier, so the fee paid will be None
-                       assert_eq!(payment_hash, our_payment_hash);
-               },
-               _ => panic!("Unexpected event")
-       }
-       match events[1] {
-               Event::PaymentPathSuccessful { payment_hash, .. } => {
-                       assert_eq!(payment_hash.unwrap(), our_payment_hash);
-               },
-               _ => panic!("Unexpected event")
-       }
-       match events[2] {
-               Event::PaymentPathSuccessful { payment_hash, .. } => {
-                       assert_eq!(payment_hash.unwrap(), our_payment_hash);
-               },
-               _ => panic!("Unexpected event")
-       }
+       expect_payment_sent(&nodes[0], our_payment_preimage, Some(None), true);
 }
 
 #[test]
@@ -9418,7 +9523,8 @@ fn test_keysend_payments_to_public_node() {
        let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 
        let test_preimage = PaymentPreimage([42; 32]);
-       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage), PaymentId(test_preimage.0)).unwrap();
+       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
+               RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -9453,7 +9559,8 @@ fn test_keysend_payments_to_private_node() {
        ).unwrap();
 
        let test_preimage = PaymentPreimage([42; 32]);
-       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage), PaymentId(test_preimage.0)).unwrap();
+       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage),
+               RecipientOnionFields::spontaneous_empty(), PaymentId(test_preimage.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -9483,7 +9590,7 @@ fn test_double_partial_claim() {
        assert_eq!(route.paths.len(), 2);
        route.paths.sort_by(|path_a, _| {
                // Sort the path so that the path through nodes[1] comes first
-               if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
+               if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
                        core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
        });
 
@@ -9501,10 +9608,11 @@ fn test_double_partial_claim() {
        ];
        expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[3], failed_destinations);
 
-       pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
+       pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
 
        // nodes[1] now retries one of the two paths...
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9575,9 +9683,11 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
        nodes[0].node.funding_transaction_generated(&temporary_channel_id, &nodes[1].node.get_our_node_id(), tx.clone()).unwrap();
        nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
        check_added_monitors!(nodes[1], 1);
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
        check_added_monitors!(nodes[0], 1);
+       expect_channel_pending_event(&nodes[0], &nodes[1].node.get_our_node_id());
 
        let (channel_ready, channel_id) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
        let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
@@ -9603,9 +9713,10 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                        // Outbound dust threshold: 2223 sats (`dust_buffer_feerate` * HTLC_TIMEOUT_TX_WEIGHT / 1000 + holder's `dust_limit_satoshis`)
                        // Outbound dust balance: 4372 sats
                        // Note, we need sent payment to be above outbound dust threshold on counterparty_tx of 2132 sats
-                       for i in 0..dust_outbound_htlc_on_holder_tx {
+                       for _ in 0..dust_outbound_htlc_on_holder_tx {
                                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], dust_outbound_htlc_on_holder_tx_msat);
-                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at dust HTLC {}", i); }
+                               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                        }
                } else {
                        // Inbound dust threshold: 2324 sats (`dust_buffer_feerate` * HTLC_SUCCESS_TX_WEIGHT / 1000 + holder's `dust_limit_satoshis`)
@@ -9619,11 +9730,12 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                if dust_outbound_balance {
                        // Outbound dust threshold: 2132 sats (`dust_buffer_feerate` * HTLC_TIMEOUT_TX_WEIGHT / 1000 + counteparty's `dust_limit_satoshis`)
                        // Outbound dust balance: 5000 sats
-                       for i in 0..dust_htlc_on_counterparty_tx {
+                       for _ in 0..dust_htlc_on_counterparty_tx {
                                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], dust_htlc_on_counterparty_tx_msat);
-                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at dust HTLC {}", i); }
+                               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                        }
-               } else {
+               } else {
                        // Inbound dust threshold: 2031 sats (`dust_buffer_feerate` * HTLC_TIMEOUT_TX_WEIGHT / 1000 + counteparty's `dust_limit_satoshis`)
                        // Inbound dust balance: 5000 sats
                        for _ in 0..dust_htlc_on_counterparty_tx {
@@ -9640,13 +9752,20 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                if on_holder_tx {
                        let dust_outbound_overflow = dust_outbound_htlc_on_holder_tx_msat * (dust_outbound_htlc_on_holder_tx + 1);
                        let dust_inbound_overflow = dust_inbound_htlc_on_holder_tx_msat * dust_inbound_htlc_on_holder_tx + dust_outbound_htlc_on_holder_tx_msat;
-                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", if dust_outbound_balance { dust_outbound_overflow } else { dust_inbound_overflow }, config.channel_config.max_dust_htlc_exposure_msat)));
+                       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
+                                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+                               ), true, APIError::ChannelUnavailable { ref err },
+                               assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", if dust_outbound_balance { dust_outbound_overflow } else { dust_inbound_overflow }, config.channel_config.max_dust_htlc_exposure_msat)));
                } else {
-                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", dust_overflow, config.channel_config.max_dust_htlc_exposure_msat)));
+                       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
+                                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+                               ), true, APIError::ChannelUnavailable { ref err },
+                               assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", dust_overflow, config.channel_config.max_dust_htlc_exposure_msat)));
                }
        } else if exposure_breach_event == ExposureEvent::AtHTLCReception {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], if on_holder_tx { dust_inbound_htlc_on_holder_tx_msat } else { dust_htlc_on_counterparty_tx_msat });
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+               nodes[1].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
                let mut events = nodes[1].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -9664,7 +9783,8 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                }
        } else if exposure_breach_event == ExposureEvent::AtUpdateFeeOutbound {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 2_500_000);
-               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at update_fee-swallowed HTLC", ); }
+               nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
                {
                        let mut feerate_lock = chanmon_cfgs[0].fee_estimator.sat_per_kw.lock().unwrap();
                        *feerate_lock = *feerate_lock * 10;
@@ -9716,8 +9836,8 @@ fn test_non_final_funding_tx() {
        assert_eq!(events.len(), 1);
        let mut tx = match events[0] {
                Event::FundingGenerationReady { ref channel_value_satoshis, ref output_script, .. } => {
-                       // Timelock the transaction _beyond_ the best client height + 2.
-                       Transaction { version: chan_id as i32, lock_time: PackedLockTime(best_height + 3), input: vec![input], output: vec![TxOut {
+                       // Timelock the transaction _beyond_ the best client height + 1.
+                       Transaction { version: chan_id as i32, lock_time: PackedLockTime(best_height + 2), input: vec![input], output: vec![TxOut {
                                value: *channel_value_satoshis, script_pubkey: output_script.clone(),
                        }]}
                },
@@ -9731,7 +9851,7 @@ fn test_non_final_funding_tx() {
                _ => panic!()
        }
 
-       // However, transaction should be accepted if it's in a +2 headroom from best block.
+       // However, transaction should be accepted if it's in a +1 headroom from best block.
        tx.lock_time = PackedLockTime(tx.lock_time.0 - 1);
        assert!(nodes[0].node.funding_transaction_generated(&temp_channel_id, &nodes[1].node.get_our_node_id(), tx.clone()).is_ok());
        get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
@@ -9837,7 +9957,8 @@ fn do_payment_with_custom_min_final_cltv_expiry(valid_delta: bool, use_user_hash
                (payment_hash, nodes[1].node.get_payment_preimage(payment_hash, payment_secret).unwrap(), payment_secret)
        };
        let route = get_route!(nodes[0], payment_parameters, recv_value, final_cltv_expiry_delta as u32).unwrap();
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
index 0c6d6f2b804bccd8cfb651c24995bf4959998a91..2d15876bf95ccad75fb90233b135ffd0fabd17ad 100644 (file)
@@ -19,14 +19,14 @@ use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use crate::ln::msgs;
 use crate::ln::msgs::MAX_VALUE_MSAT;
 use crate::util::chacha20::ChaCha20;
-use crate::util::crypto::hkdf_extract_expand_thrice;
+use crate::util::crypto::hkdf_extract_expand_4x;
 use crate::util::errors::APIError;
 use crate::util::logger::Logger;
 
-use core::convert::TryInto;
+use core::convert::{TryFrom, TryInto};
 use core::ops::Deref;
 
-const IV_LEN: usize = 16;
+pub(crate) const IV_LEN: usize = 16;
 const METADATA_LEN: usize = 16;
 const METADATA_KEY_LEN: usize = 32;
 const AMT_MSAT_LEN: usize = 8;
@@ -48,6 +48,8 @@ pub struct ExpandedKey {
        /// The key used to authenticate a user-provided payment hash and metadata as previously
        /// registered with LDK.
        user_pmt_hash_key: [u8; 32],
+       /// The base key used to derive signing keys and authenticate messages for BOLT 12 Offers.
+       offers_base_key: [u8; 32],
 }
 
 impl ExpandedKey {
@@ -55,14 +57,76 @@ impl ExpandedKey {
        ///
        /// It is recommended to cache this value and not regenerate it for each new inbound payment.
        pub fn new(key_material: &KeyMaterial) -> ExpandedKey {
-               let (metadata_key, ldk_pmt_hash_key, user_pmt_hash_key) =
-                       hkdf_extract_expand_thrice(b"LDK Inbound Payment Key Expansion", &key_material.0);
+               let (metadata_key, ldk_pmt_hash_key, user_pmt_hash_key, offers_base_key) =
+                       hkdf_extract_expand_4x(b"LDK Inbound Payment Key Expansion", &key_material.0);
                Self {
                        metadata_key,
                        ldk_pmt_hash_key,
                        user_pmt_hash_key,
+                       offers_base_key,
                }
        }
+
+       /// Returns an [`HmacEngine`] used to construct [`Offer::metadata`].
+       ///
+       /// [`Offer::metadata`]: crate::offers::offer::Offer::metadata
+       #[allow(unused)]
+       pub(crate) fn hmac_for_offer(
+               &self, nonce: Nonce, iv_bytes: &[u8; IV_LEN]
+       ) -> HmacEngine<Sha256> {
+               let mut hmac = HmacEngine::<Sha256>::new(&self.offers_base_key);
+               hmac.input(iv_bytes);
+               hmac.input(&nonce.0);
+               hmac
+       }
+}
+
+/// A 128-bit number used only once.
+///
+/// Needed when constructing [`Offer::metadata`] and deriving [`Offer::signing_pubkey`] from
+/// [`ExpandedKey`]. Must not be reused for any other derivation without first hashing.
+///
+/// [`Offer::metadata`]: crate::offers::offer::Offer::metadata
+/// [`Offer::signing_pubkey`]: crate::offers::offer::Offer::signing_pubkey
+#[allow(unused)]
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub(crate) struct Nonce(pub(crate) [u8; Self::LENGTH]);
+
+impl Nonce {
+       /// Number of bytes in the nonce.
+       pub const LENGTH: usize = 16;
+
+       /// Creates a `Nonce` from the given [`EntropySource`].
+       pub fn from_entropy_source<ES: Deref>(entropy_source: ES) -> Self
+       where
+               ES::Target: EntropySource,
+       {
+               let mut bytes = [0u8; Self::LENGTH];
+               let rand_bytes = entropy_source.get_secure_random_bytes();
+               bytes.copy_from_slice(&rand_bytes[..Self::LENGTH]);
+
+               Nonce(bytes)
+       }
+
+       /// Returns a slice of the underlying bytes of size [`Nonce::LENGTH`].
+       pub fn as_slice(&self) -> &[u8] {
+               &self.0
+       }
+}
+
+impl TryFrom<&[u8]> for Nonce {
+       type Error = ();
+
+       fn try_from(bytes: &[u8]) -> Result<Self, ()> {
+               if bytes.len() != Self::LENGTH {
+                       return Err(());
+               }
+
+               let mut copied_bytes = [0u8; Self::LENGTH];
+               copied_bytes.copy_from_slice(bytes);
+
+               Ok(Self(copied_bytes))
+       }
 }
 
 enum Method {
index 5bd2e87ba5c15ca3f1ef998bafc15b92e5c88c68..b20a2e57526b6f3a50596477a5097496172f57a9 100644 (file)
@@ -24,15 +24,13 @@ use crate::ln::channel;
 use crate::ln::chan_utils;
 #[cfg(anchors)]
 use crate::ln::channelmanager::ChannelManager;
-use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, PaymentId};
+use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, PaymentId, RecipientOnionFields};
 use crate::ln::msgs::ChannelMessageHandler;
 #[cfg(anchors)]
 use crate::util::config::UserConfig;
 #[cfg(anchors)]
 use crate::util::crypto::sign;
-#[cfg(anchors)]
 use crate::util::ser::Writeable;
-#[cfg(anchors)]
 use crate::util::test_utils;
 
 #[cfg(anchors)]
@@ -78,7 +76,8 @@ fn chanmon_fail_from_stale_commitment() {
        let (update_a, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2);
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let bs_txn = get_local_commitment_txn!(nodes[1], chan_id_2);
@@ -498,7 +497,7 @@ fn do_test_claim_value_force_close(prev_commitment_tx: bool) {
                nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances());
 
        // When the HTLC timeout output is spendable in the next block, A should broadcast it
-       connect_blocks(&nodes[0], htlc_cltv_timeout - nodes[0].best_block_info().1 - 1);
+       connect_blocks(&nodes[0], htlc_cltv_timeout - nodes[0].best_block_info().1);
        let a_broadcast_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(a_broadcast_txn.len(), 2);
        assert_eq!(a_broadcast_txn[0].input.len(), 1);
@@ -625,7 +624,8 @@ fn test_balances_on_local_commitment_htlcs() {
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 10_000_000);
        let htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -636,7 +636,8 @@ fn test_balances_on_local_commitment_htlcs() {
        expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 10_000_000);
 
        let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 20_000_000);
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -886,7 +887,7 @@ fn test_no_preimage_inbound_htlc_balances() {
        // HTLC has been spent, even after the HTLC expires. We'll also fail the inbound HTLC, but it
        // won't do anything as the channel is already closed.
 
-       connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1);
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV);
        let as_htlc_timeout_claim = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(as_htlc_timeout_claim.len(), 1);
        check_spends!(as_htlc_timeout_claim[0], as_txn[0]);
@@ -907,7 +908,7 @@ fn test_no_preimage_inbound_htlc_balances() {
 
        // The next few blocks for B look the same as for A, though for the opposite HTLC
        nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
-       connect_blocks(&nodes[1], TEST_FINAL_CLTV - (ANTI_REORG_DELAY - 1) - 1);
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV - (ANTI_REORG_DELAY - 1));
        expect_pending_htlcs_forwardable_conditions!(nodes[1],
                [HTLCDestination::FailedPayment { payment_hash: to_b_failed_payment_hash }]);
        let bs_htlc_timeout_claim = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
@@ -1321,20 +1322,29 @@ fn test_revoked_counterparty_htlc_tx_balances() {
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
-       let revoked_htlc_success_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-
-       assert_eq!(revoked_htlc_success_txn.len(), 1);
-       assert_eq!(revoked_htlc_success_txn[0].input.len(), 1);
-       assert_eq!(revoked_htlc_success_txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
-       check_spends!(revoked_htlc_success_txn[0], revoked_local_txn[0]);
+       let revoked_htlc_success = {
+               let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               assert_eq!(txn[0].input.len(), 1);
+               assert_eq!(txn[0].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
+               check_spends!(txn[0], revoked_local_txn[0]);
+               txn.pop().unwrap()
+       };
 
        connect_blocks(&nodes[1], TEST_FINAL_CLTV);
-       let revoked_htlc_timeout_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(revoked_htlc_timeout_txn.len(), 1);
-       check_spends!(revoked_htlc_timeout_txn[0], revoked_local_txn[0]);
-       assert_ne!(revoked_htlc_success_txn[0].input[0].previous_output, revoked_htlc_timeout_txn[0].input[0].previous_output);
-       assert_eq!(revoked_htlc_success_txn[0].lock_time.0, 0);
-       assert_ne!(revoked_htlc_timeout_txn[0].lock_time.0, 0);
+       let revoked_htlc_timeout = {
+               let mut txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
+               if txn[0].input[0].previous_output == revoked_htlc_success.input[0].previous_output {
+                       txn.remove(1)
+               } else {
+                       txn.remove(0)
+               }
+       };
+       check_spends!(revoked_htlc_timeout, revoked_local_txn[0]);
+       assert_ne!(revoked_htlc_success.input[0].previous_output, revoked_htlc_timeout.input[0].previous_output);
+       assert_eq!(revoked_htlc_success.lock_time.0, 0);
+       assert_ne!(revoked_htlc_timeout.lock_time.0, 0);
 
        // A will generate justice tx from B's revoked commitment/HTLC tx
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
@@ -1366,10 +1376,10 @@ fn test_revoked_counterparty_htlc_tx_balances() {
        assert_eq!(as_balances,
                sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()));
 
-       mine_transaction(&nodes[0], &revoked_htlc_success_txn[0]);
+       mine_transaction(&nodes[0], &revoked_htlc_success);
        let as_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(as_htlc_claim_tx.len(), 2);
-       check_spends!(as_htlc_claim_tx[0], revoked_htlc_success_txn[0]);
+       check_spends!(as_htlc_claim_tx[0], revoked_htlc_success);
        check_spends!(as_htlc_claim_tx[1], revoked_local_txn[0]); // A has to generate a new claim for the remaining revoked
                                                                  // outputs (which no longer includes the spent HTLC output)
 
@@ -1378,7 +1388,7 @@ fn test_revoked_counterparty_htlc_tx_balances() {
 
        assert_eq!(as_htlc_claim_tx[0].output.len(), 1);
        fuzzy_assert_eq(as_htlc_claim_tx[0].output[0].value,
-               3_000 - chan_feerate * (revoked_htlc_success_txn[0].weight() + as_htlc_claim_tx[0].weight()) as u64 / 1000);
+               3_000 - chan_feerate * (revoked_htlc_success.weight() + as_htlc_claim_tx[0].weight()) as u64 / 1000);
 
        mine_transaction(&nodes[0], &as_htlc_claim_tx[0]);
        assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
@@ -1420,7 +1430,7 @@ fn test_revoked_counterparty_htlc_tx_balances() {
                }]),
                sorted_vec(nodes[0].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances()));
 
-       connect_blocks(&nodes[0], revoked_htlc_timeout_txn[0].lock_time.0 - nodes[0].best_block_info().1);
+       connect_blocks(&nodes[0], revoked_htlc_timeout.lock_time.0 - nodes[0].best_block_info().1);
        expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(&nodes[0],
                [HTLCDestination::FailedPayment { payment_hash: failed_payment_hash }]);
        // As time goes on A may split its revocation claim transaction into multiple.
@@ -1437,11 +1447,11 @@ fn test_revoked_counterparty_htlc_tx_balances() {
                check_spends!(tx, revoked_local_txn[0]);
        }
 
-       mine_transaction(&nodes[0], &revoked_htlc_timeout_txn[0]);
+       mine_transaction(&nodes[0], &revoked_htlc_timeout);
        let as_second_htlc_claim_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(as_second_htlc_claim_tx.len(), 2);
 
-       check_spends!(as_second_htlc_claim_tx[0], revoked_htlc_timeout_txn[0]);
+       check_spends!(as_second_htlc_claim_tx[0], revoked_htlc_timeout);
        check_spends!(as_second_htlc_claim_tx[1], revoked_local_txn[0]);
 
        // Connect blocks to finalize the HTLC resolution with the HTLC-Timeout transaction. In a
@@ -1692,6 +1702,233 @@ fn test_revoked_counterparty_aggregated_claims() {
        assert!(nodes[1].chain_monitor.chain_monitor.get_monitor(funding_outpoint).unwrap().get_claimable_balances().is_empty());
 }
 
+fn do_test_restored_packages_retry(check_old_monitor_retries_after_upgrade: bool) {
+       // Tests that we'll retry packages that were previously timelocked after we've restored them.
+       let persister;
+       let new_chain_monitor;
+       let node_deserialized;
+
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       // Open a channel, lock in an HTLC, and immediately broadcast the commitment transaction. This
+       // ensures that the HTLC timeout package is held until we reach its expiration height.
+       let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 50_000_000);
+       route_payment(&nodes[0], &[&nodes[1]], 10_000_000);
+
+       nodes[0].node.force_close_broadcasting_latest_txn(&chan_id, &nodes[1].node.get_our_node_id()).unwrap();
+       check_added_monitors(&nodes[0], 1);
+       check_closed_broadcast(&nodes[0], 1, true);
+       check_closed_event(&nodes[0], 1, ClosureReason::HolderForceClosed, false);
+
+       let commitment_tx = {
+               let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               assert_eq!(txn[0].output.len(), 3);
+               check_spends!(txn[0], funding_tx);
+               txn.pop().unwrap()
+       };
+
+       mine_transaction(&nodes[0], &commitment_tx);
+
+       // Connect blocks until the HTLC's expiration is met, expecting a transaction broadcast.
+       connect_blocks(&nodes[0], TEST_FINAL_CLTV);
+       let htlc_timeout_tx = {
+               let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+               assert_eq!(txn.len(), 1);
+               check_spends!(txn[0], commitment_tx);
+               txn.pop().unwrap()
+       };
+
+       // Check that we can still rebroadcast these packages/transactions if we're upgrading from an
+       // old `ChannelMonitor` that did not exercise said rebroadcasting logic.
+       if check_old_monitor_retries_after_upgrade {
+               let serialized_monitor = hex::decode(
+                       "0101fffffffffffffffff9550f22c95100160014d5a9aa98b89acc215fc3d23d6fec0ad59ca3665f00002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6302d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e2035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea0016001467822698d782e8421ebdf96d010de99382b7ec2300160014caf6d80fe2bab80473b021f57588a9c384bf23170000000000000000000000004d49e5da0000000000000000000000000000002a0270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f74c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000022002034c0cc0ad0dd5fe61dcf7ef58f995e3d34f8dbd24aa2a6fae68fefe102bf025c21391732ce658e1fe167300bb689a81e7db5399b9ee4095e217b0e997e8dd3d17a0000000000000000004a002103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84022103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a04020090004752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae00000000000186a0ffffffffffff0291e7c0a3232fb8650a6b4089568a81062b48a768780e5a74bb4a4a74e33aec2c029d5760248ec86c4a76d9df8308555785a06a65472fb995f5b392d520bbd000650090c1c94b11625690c9d84c5daa67b6ad19fcc7f9f23e194384140b08fcab9e8e810000ffffffffffff000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000002391732ce658e1fe167300bb689a81e7db5399b9ee4095e217b0e997e8dd3d17a00000000000000010000000000009896800000005166687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f292505000000009c009900202d704fbfe342a9ff6eaca14d80a24aaed0e680bbbdd36157b6f2798c61d906910120f9fe5e552aa0fc45020f0505efde432a4e373e5d393863973a6899f8c26d33d10208000000000098968004494800210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23020500030241000408000001000000000006020000080800000000009896800a0400000046167c86cc0e598a6b541f7c9bf9ef17222e4a76f636e2d22185aeadd2b02d029c00000000000000000000000000000000000000000000000166687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925fffffffffffe01e3002004f8eda5676356f539169a8e9a1e86c7f125283328d6f4bded1b939b52a6a7e30108000000000000c299022103a1f98e85886df54add6908b4fc1ff515e44aedefe9eb9c02879c89994298fa79042103a650bf03971df0176c7b412247390ef717853e8bd487b204dccc2fe2078bb75206210390bbbcebe9f70ba5dfd98866a79f72f75e0a6ea550ef73b202dd87cd6477350a08210284152d57908488e666e872716a286eb670b3d06cbeebf3f2e4ad350e01ec5e5b0a2102295e2de39eb3dcc2882f8cc266df7882a8b6d2c32aa08799f49b693aad3be28e0c04000000fd0e00fd01fe002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01080000000000009b5e0221035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea04210230fde9c031f487db95ff55b7c0acbe0c7c26a8d82615e9184416bd350101616706210225afb4e88eac8b47b67adeaf085f5eb5d37d936f56138f0848de3d104edf113208210208e4687a95c172b86b920c3bc5dbd5f023094ec2cb0abdb74f9b624f45740df90a2102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e20c04000000fd0efd01193b00010102080000000000989680040400000051062066687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925080400000000417e2650c201383711eed2a7cb8652c3e77ee6a395e81849c5c222217ed68b333c0ca9f1e900662ae68a7359efa7ef9d90613f2a62f7c3ff90f8c25e2cc974c9d39c009900202d704fbfe342a9ff6eaca14d80a24aaed0e680bbbdd36157b6f2798c61d906910120f9fe5e552aa0fc45020f0505efde432a4e373e5d393863973a6899f8c26d33d10208000000000098968004494800210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c23020500030241000408000001000000000006020000080800000000009896800a0400000046fffffffffffefffffffffffe000000000000000000000000000000000000000000000000ffe099e83ae3761c7f1b781d22613bd1f6977e9ad59fae12b3eba34462ee8a3d000000500000000000000002fd01da002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01fd01840200000000010174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d304004730440220671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec602204b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613401473044022016a0da36f70cbf5d889586af88f238982889dc161462c56557125c7acfcb69e9022036ae10c6cc8cbc3b27d9e9ef6babb556086585bc819f252208bd175286699fdd014752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae50c9222002040000000b03209452ca8c90d4c78928b80ec41398f2a890324d8ad6e6c81408a0cb9b8d977b070406030400020090fd02a1002045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d01fd01840200000000010174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d304004730440220671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec602204b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613401473044022016a0da36f70cbf5d889586af88f238982889dc161462c56557125c7acfcb69e9022036ae10c6cc8cbc3b27d9e9ef6babb556086585bc819f252208bd175286699fdd014752210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db32103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b52ae50c9222002040000000b03209452ca8c90d4c78928b80ec41398f2a890324d8ad6e6c81408a0cb9b8d977b0704cd01cb00c901c7002245cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d0001022102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e204020090062b5e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c630821035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea0a200000000000000000000000004d49e5da0000000000000000000000000000002a0c0800000000000186a0000000000000000274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e0000000000000001000000000022002034c0cc0ad0dd5fe61dcf7ef58f995e3d34f8dbd24aa2a6fae68fefe102bf025c45cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d000000000000000100000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d5349010100160014d5a9aa98b89acc215fc3d23d6fec0ad59ca3665ffd027100fd01e6fd01e300080000fffffffffffe02080000000000009b5e0408000000000000c3500604000000fd08b0af002102d7dde8e10a5a22c9bd0d7ef5494d85683ac050253b917615d4f97af633f0a8e20221035f5e9d58b4328566223c107d86cf853e6b9fae1d26ff6d969be0178d1423c4ea04210230fde9c031f487db95ff55b7c0acbe0c7c26a8d82615e9184416bd350101616706210225afb4e88eac8b47b67adeaf085f5eb5d37d936f56138f0848de3d104edf113208210208e4687a95c172b86b920c3bc5dbd5f023094ec2cb0abdb74f9b624f45740df90acdcc00a8020000000174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800310270000000000002200208309b406e3b96e76cde414fbb8f5159f5b25b24075656c6382cec797854d53495e9b0000000000002200204c5f18e5e95b184f34d02ba6de8a2a4e36ae3d4ec87299ad81f3284dc7195c6350c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d350c92220022045cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d0c3c3b00010102080000000000989680040400000051062066687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f29250804000000000240671c9badf26bd3a1ebd2d17020c6be20587d7822530daacc52c28839875eaec64b575a21729ed27311f6d79fdf6fe8702b0a798f7d842e39ede1b56f249a613404010006407e2650c201383711eed2a7cb8652c3e77ee6a395e81849c5c222217ed68b333c0ca9f1e900662ae68a7359efa7ef9d90613f2a62f7c3ff90f8c25e2cc974c9d3010000000000000001010000000000000000090b2a953d93a124c600ecb1a0ccfed420169cdd37f538ad94a3e4e6318c93c14adf59cdfbb40bdd40950c9f8dd547d29d75a173e1376a7850743394c46dea2dfd01cefd01ca00fd017ffd017c00080000ffffffffffff0208000000000000c2990408000000000000c3500604000000fd08b0af002102295e2de39eb3dcc2882f8cc266df7882a8b6d2c32aa08799f49b693aad3be28e022103a1f98e85886df54add6908b4fc1ff515e44aedefe9eb9c02879c89994298fa79042103a650bf03971df0176c7b412247390ef717853e8bd487b204dccc2fe2078bb75206210390bbbcebe9f70ba5dfd98866a79f72f75e0a6ea550ef73b202dd87cd6477350a08210284152d57908488e666e872716a286eb670b3d06cbeebf3f2e4ad350e01ec5e5b0aa2a1007d020000000174c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e00000000000f55f9800299c2000000000000220020740e108cfbc93967b6ab242a351ebee7de51814cf78d366adefd78b10281f17e50c300000000000016001425df8ec4a074f80579fed67d4707d5ec8ed7e8d351c92220022004f8eda5676356f539169a8e9a1e86c7f125283328d6f4bded1b939b52a6a7e30c00024045cb2485594bb1ec08e7bb6af4f89c912bd53f006d7876ea956773e04a4aad4a40e2b8d4fc612102f0b54061b3c1239fb78783053e8e6f9d92b1b99f81ae9ec2040100060000fd019600b0af002103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b02210270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f042103b4e59df102747edc3a3e2283b42b88a8c8218ffd0dcfb52f2524b371d64cadaa062103d902b7b8b3434076d2b210e912c76645048b71e28995aad227a465a65ccd817608210301e9a52f923c157941de4a7692e601f758660969dcf5abdb67817efe84cce2ef0202009004010106b7b600b0af00210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db30221034d0f817cb19b4a3bd144b615459bd06cbab3b4bdc96d73e18549a992cee80e8104210380542b59a9679890cba529fe155a9508ef57dac7416d035b23666e3fb98c3814062103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84082103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a02020090082274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e000000000287010108d30df34e3a1e00ecdd03a2c843db062479a81752c4dfd0cc4baef0f81e7bc7ef8820990daf8d8e8d30a3b4b08af12c9f5cd71e45c7238103e0c80ca13850862e4fd2c56b69b7195312518de1bfe9aed63c80bb7760d70b2a870d542d815895fd12423d11e2adb0cdf55d776dac8f487c9b3b7ea12f1b150eb15889cf41333ade465692bf1cdc360b9c2a19bf8c1ca4fed7639d8bc953d36c10d8c6c9a8c0a57608788979bcf145e61b308006896e21d03e92084f93bd78740c20639134a7a8fd019afd019600b0af002103c21e841cbc0b48197d060c71e116c185fa0ac281b7d0aa5924f535154437ca3b02210270b20ad0f2c2bb30a55590fc77778495bc1b38c96476901145dda57491237f0f042103b4e59df102747edc3a3e2283b42b88a8c8218ffd0dcfb52f2524b371d64cadaa062103d902b7b8b3434076d2b210e912c76645048b71e28995aad227a465a65ccd817608210301e9a52f923c157941de4a7692e601f758660969dcf5abdb67817efe84cce2ef0202009004010106b7b600b0af00210307a78def56cba9fc4db22a25928181de538ee59ba1a475ae113af7790acd0db30221034d0f817cb19b4a3bd144b615459bd06cbab3b4bdc96d73e18549a992cee80e8104210380542b59a9679890cba529fe155a9508ef57dac7416d035b23666e3fb98c3814062103adde8029d3ee281a32e9db929b39f503ff9d7e93cd308eb157955344dc6def84082103205087e2dc1f6b9937e887dfa712c5bdfa950b01dbda3ebac4c85efdde48ee6a02020090082274c52ab4f11296d62b66a6dba9513b04a3e7fb5a09a30cee22fce7294ab55b7e000000000000000186a00000000000000000000000004d49e5da0000000000000000000000000000002a000000000000000001b77b61346a2a408afdb01743a2230cb36e55771a0790f67a0910e207fd223fc8000000000000000145cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d00000000041000080000000000989680020400000051160004000000510208000000000000000004040000000b000000000000000145cfd42d0989e55b953f516ac7fd152bd90ec4438a2fc636f97ddd32a0c8fe0d00000000b77b61346a2a408afdb01743a2230cb36e55771a0790f67a0910e207fd223fc80000005000000000000000000000000000000000000101300300050007010109210355f8d2238a322d16b602bd0ceaad5b01019fb055971eaadcc9b29226a4da6c230d000f020000",
+               ).unwrap();
+               reload_node!(nodes[0], &nodes[0].node.encode(), &[&serialized_monitor], persister, new_chain_monitor, node_deserialized);
+       }
+
+       // Connecting more blocks should result in the HTLC transactions being rebroadcast.
+       connect_blocks(&nodes[0], 6);
+       if check_old_monitor_retries_after_upgrade {
+               check_added_monitors(&nodes[0], 1);
+       }
+       {
+               let txn = nodes[0].tx_broadcaster.txn_broadcast();
+               if !nodes[0].connect_style.borrow().skips_blocks() {
+                       assert_eq!(txn.len(), 6);
+               } else {
+                       assert!(txn.len() < 6);
+               }
+               for tx in txn {
+                       assert_eq!(tx.input.len(), htlc_timeout_tx.input.len());
+                       assert_eq!(tx.output.len(), htlc_timeout_tx.output.len());
+                       assert_eq!(tx.input[0].previous_output, htlc_timeout_tx.input[0].previous_output);
+                       assert_eq!(tx.output[0], htlc_timeout_tx.output[0]);
+               }
+       }
+}
+
+#[test]
+fn test_restored_packages_retry() {
+       do_test_restored_packages_retry(false);
+       do_test_restored_packages_retry(true);
+}
+
+fn do_test_monitor_rebroadcast_pending_claims(anchors: bool) {
+       // Test that we will retry broadcasting pending claims for a force-closed channel on every
+       // `ChainMonitor::rebroadcast_pending_claims` call.
+       if anchors {
+               assert!(cfg!(anchors));
+       }
+       let mut chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let mut config = test_default_channel_config();
+       if anchors {
+               #[cfg(anchors)] {
+                       config.channel_handshake_config.negotiate_anchors_zero_fee_htlc_tx = true;
+               }
+       }
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[Some(config), Some(config)]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, _, chan_id, funding_tx) = create_chan_between_nodes_with_value(
+               &nodes[0], &nodes[1], 1_000_000, 500_000_000
+       );
+       const HTLC_AMT_MSAT: u64 = 1_000_000;
+       const HTLC_AMT_SAT: u64 = HTLC_AMT_MSAT / 1000;
+       route_payment(&nodes[0], &[&nodes[1]], HTLC_AMT_MSAT);
+
+       let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1;
+
+       let commitment_txn = get_local_commitment_txn!(&nodes[0], &chan_id);
+       assert_eq!(commitment_txn.len(), if anchors { 1 /* commitment tx only */} else { 2 /* commitment and htlc timeout tx */ });
+       check_spends!(&commitment_txn[0], &funding_tx);
+       mine_transaction(&nodes[0], &commitment_txn[0]);
+       check_closed_broadcast!(&nodes[0], true);
+       check_closed_event(&nodes[0], 1, ClosureReason::CommitmentTxConfirmed, false);
+       check_added_monitors(&nodes[0], 1);
+
+       // Set up a helper closure we'll use throughout our test. We should only expect retries without
+       // bumps if fees have not increased after a block has been connected (assuming the height timer
+       // re-evaluates at every block) or after `ChainMonitor::rebroadcast_pending_claims` is called.
+       let mut prev_htlc_tx_feerate = None;
+       let mut check_htlc_retry = |should_retry: bool, should_bump: bool| -> Option<Transaction> {
+               let (htlc_tx, htlc_tx_feerate) = if anchors {
+                       assert!(nodes[0].tx_broadcaster.txn_broadcast().is_empty());
+                       let mut events = nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events();
+                       assert_eq!(events.len(), if should_retry { 1 } else { 0 });
+                       if !should_retry {
+                               return None;
+                       }
+                       #[allow(unused_assignments)]
+                       let mut tx = Transaction {
+                               version: 2,
+                               lock_time: bitcoin::PackedLockTime::ZERO,
+                               input: vec![],
+                               output: vec![],
+                       };
+                       #[allow(unused_assignments)]
+                       let mut feerate = 0;
+                       #[cfg(anchors)] {
+                               feerate = if let Event::BumpTransaction(BumpTransactionEvent::HTLCResolution {
+                                       target_feerate_sat_per_1000_weight, mut htlc_descriptors, tx_lock_time,
+                               }) = events.pop().unwrap() {
+                                       let secp = Secp256k1::new();
+                                       assert_eq!(htlc_descriptors.len(), 1);
+                                       let descriptor = htlc_descriptors.pop().unwrap();
+                                       assert_eq!(descriptor.commitment_txid, commitment_txn[0].txid());
+                                       let htlc_output_idx = descriptor.htlc.transaction_output_index.unwrap() as usize;
+                                       assert!(htlc_output_idx < commitment_txn[0].output.len());
+                                       tx.lock_time = tx_lock_time;
+                                       // Note that we don't care about actually making the HTLC transaction meet the
+                                       // feerate for the test, we just want to make sure the feerates we receive from
+                                       // the events never decrease.
+                                       tx.input.push(descriptor.unsigned_tx_input());
+                                       let signer = nodes[0].keys_manager.derive_channel_keys(
+                                               descriptor.channel_value_satoshis, &descriptor.channel_keys_id,
+                                       );
+                                       let per_commitment_point = signer.get_per_commitment_point(
+                                               descriptor.per_commitment_number, &secp
+                                       );
+                                       tx.output.push(descriptor.tx_output(&per_commitment_point, &secp));
+                                       let our_sig = signer.sign_holder_htlc_transaction(&mut tx, 0, &descriptor, &secp).unwrap();
+                                       let witness_script = descriptor.witness_script(&per_commitment_point, &secp);
+                                       tx.input[0].witness = descriptor.tx_input_witness(&our_sig, &witness_script);
+                                       target_feerate_sat_per_1000_weight as u64
+                               } else { panic!("unexpected event"); };
+                       }
+                       (tx, feerate)
+               } else {
+                       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+                       let mut txn = nodes[0].tx_broadcaster.txn_broadcast();
+                       assert_eq!(txn.len(), if should_retry { 1 } else { 0 });
+                       if !should_retry {
+                               return None;
+                       }
+                       let htlc_tx = txn.pop().unwrap();
+                       check_spends!(htlc_tx, commitment_txn[0]);
+                       let htlc_tx_fee = HTLC_AMT_SAT - htlc_tx.output[0].value;
+                       let htlc_tx_feerate = htlc_tx_fee * 1000 / htlc_tx.weight() as u64;
+                       (htlc_tx, htlc_tx_feerate)
+               };
+               if should_bump {
+                       assert!(htlc_tx_feerate > prev_htlc_tx_feerate.take().unwrap());
+               } else if let Some(prev_feerate) = prev_htlc_tx_feerate.take() {
+                       assert_eq!(htlc_tx_feerate, prev_feerate);
+               }
+               prev_htlc_tx_feerate = Some(htlc_tx_feerate);
+               Some(htlc_tx)
+       };
+
+       // Connect blocks up to one before the HTLC expires. This should not result in a claim/retry.
+       connect_blocks(&nodes[0], htlc_expiry - nodes[0].best_block_info().1 - 1);
+       check_htlc_retry(false, false);
+
+       // Connect one more block, producing our first claim.
+       connect_blocks(&nodes[0], 1);
+       check_htlc_retry(true, false);
+
+       // Connect one more block, expecting a retry with a fee bump. Unfortunately, we cannot bump HTLC
+       // transactions pre-anchors.
+       connect_blocks(&nodes[0], 1);
+       check_htlc_retry(true, anchors);
+
+       // Trigger a call and we should have another retry, but without a bump.
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(true, false);
+
+       // Double the feerate and trigger a call, expecting a fee-bumped retry.
+       *nodes[0].fee_estimator.sat_per_kw.lock().unwrap() *= 2;
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(true, anchors);
+
+       // Connect one more block, expecting a retry with a fee bump. Unfortunately, we cannot bump HTLC
+       // transactions pre-anchors.
+       connect_blocks(&nodes[0], 1);
+       let htlc_tx = check_htlc_retry(true, anchors).unwrap();
+
+       // Mine the HTLC transaction to ensure we don't retry claims while they're confirmed.
+       mine_transaction(&nodes[0], &htlc_tx);
+       // If we have a `ConnectStyle` that advertises the new block first without the transasctions,
+       // we'll receive an extra bumped claim.
+       if nodes[0].connect_style.borrow().updates_best_block_first() {
+               check_htlc_retry(true, anchors);
+       }
+       nodes[0].chain_monitor.chain_monitor.rebroadcast_pending_claims();
+       check_htlc_retry(false, false);
+}
+
+#[test]
+fn test_monitor_timer_based_claim() {
+       do_test_monitor_rebroadcast_pending_claims(false);
+       #[cfg(anchors)]
+       do_test_monitor_rebroadcast_pending_claims(true);
+}
+
 #[cfg(anchors)]
 #[test]
 fn test_yield_anchors_events() {
index cb857ed5ee6004d9e98d96c935c253f835d30d2e..4b2eb9674fa8acefa0afba97245ca4674d49e577 100644 (file)
@@ -42,15 +42,20 @@ use crate::io_extras::read_to_end;
 
 use crate::events::{MessageSendEventsProvider, OnionMessageProvider};
 use crate::util::logger;
-use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
+use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, WithoutLength, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
 
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 
-use crate::routing::gossip::NodeId;
+use crate::routing::gossip::{NodeAlias, NodeId};
 
 /// 21 million * 10^8 * 1000
 pub(crate) const MAX_VALUE_MSAT: u64 = 21_000_000_0000_0000_000;
 
+#[cfg(taproot)]
+/// A partial signature that also contains the Musig2 nonce its signer used
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct PartialSignatureWithNonce(pub musig2::types::PartialSignature, pub musig2::types::PublicNonce);
+
 /// An error in decoding a message or struct.
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub enum DecodeError {
@@ -244,6 +249,9 @@ pub struct AcceptChannel {
        /// our feature bits with our counterparty's feature bits from the [`Init`] message.
        /// This is required to match the equivalent field in [`OpenChannel::channel_type`].
        pub channel_type: Option<ChannelTypeFeatures>,
+       #[cfg(taproot)]
+       /// Next nonce the channel initiator should use to create a funding output signature against
+       pub next_local_nonce: Option<musig2::types::PublicNonce>,
 }
 
 /// A [`funding_created`] message to be sent to or received from a peer.
@@ -259,6 +267,12 @@ pub struct FundingCreated {
        pub funding_output_index: u16,
        /// The signature of the channel initiator (funder) on the initial commitment transaction
        pub signature: Signature,
+       #[cfg(taproot)]
+       /// The partial signature of the channel initiator (funder)
+       pub partial_signature_with_nonce: Option<PartialSignatureWithNonce>,
+       #[cfg(taproot)]
+       /// Next nonce the channel acceptor should use to finalize the funding output signature
+       pub next_local_nonce: Option<musig2::types::PublicNonce>
 }
 
 /// A [`funding_signed`] message to be sent to or received from a peer.
@@ -270,6 +284,9 @@ pub struct FundingSigned {
        pub channel_id: [u8; 32],
        /// The signature of the channel acceptor (fundee) on the initial commitment transaction
        pub signature: Signature,
+       #[cfg(taproot)]
+       /// The partial signature of the channel acceptor (fundee)
+       pub partial_signature_with_nonce: Option<PartialSignatureWithNonce>,
 }
 
 /// A [`channel_ready`] message to be sent to or received from a peer.
@@ -409,6 +426,9 @@ pub struct CommitmentSigned {
        pub signature: Signature,
        /// Signatures on the HTLC transactions
        pub htlc_signatures: Vec<Signature>,
+       #[cfg(taproot)]
+       /// The partial Taproot signature on the commitment transaction
+       pub partial_signature_with_nonce: Option<PartialSignatureWithNonce>,
 }
 
 /// A [`revoke_and_ack`] message to be sent to or received from a peer.
@@ -422,6 +442,9 @@ pub struct RevokeAndACK {
        pub per_commitment_secret: [u8; 32],
        /// The next sender-broadcast commitment transaction's per-commitment point
        pub next_per_commitment_point: PublicKey,
+       #[cfg(taproot)]
+       /// Musig nonce the recipient should use in their next commitment signature message
+       pub next_local_nonce: Option<musig2::types::PublicNonce>
 }
 
 /// An [`update_fee`] message to be sent to or received from a peer
@@ -671,7 +694,7 @@ pub struct UnsignedNodeAnnouncement {
        /// An alias, for UI purposes.
        ///
        /// This should be sanitized before use. There is no guarantee of uniqueness.
-       pub alias: [u8; 32],
+       pub alias: NodeAlias,
        /// List of addresses on which this node is reachable
        pub addresses: Vec<NetAddress>,
        pub(crate) excess_address_data: Vec<u8>,
@@ -1145,6 +1168,7 @@ mod fuzzy_internal_msgs {
                },
                FinalNode {
                        payment_data: Option<FinalOnionHopData>,
+                       payment_metadata: Option<Vec<u8>>,
                        keysend_preimage: Option<PaymentPreimage>,
                },
        }
@@ -1288,7 +1312,28 @@ impl Readable for OptionalField<u64> {
        }
 }
 
+#[cfg(not(taproot))]
+impl_writeable_msg!(AcceptChannel, {
+       temporary_channel_id,
+       dust_limit_satoshis,
+       max_htlc_value_in_flight_msat,
+       channel_reserve_satoshis,
+       htlc_minimum_msat,
+       minimum_depth,
+       to_self_delay,
+       max_accepted_htlcs,
+       funding_pubkey,
+       revocation_basepoint,
+       payment_point,
+       delayed_payment_basepoint,
+       htlc_basepoint,
+       first_per_commitment_point,
+       shutdown_scriptpubkey
+}, {
+       (1, channel_type, option),
+});
 
+#[cfg(taproot)]
 impl_writeable_msg!(AcceptChannel, {
        temporary_channel_id,
        dust_limit_satoshis,
@@ -1307,6 +1352,7 @@ impl_writeable_msg!(AcceptChannel, {
        shutdown_scriptpubkey
 }, {
        (1, channel_type, option),
+       (4, next_local_nonce, option),
 });
 
 impl_writeable_msg!(AnnouncementSignatures, {
@@ -1363,30 +1409,60 @@ impl_writeable!(ClosingSignedFeeRange, {
        max_fee_satoshis
 });
 
+#[cfg(not(taproot))]
 impl_writeable_msg!(CommitmentSigned, {
        channel_id,
        signature,
        htlc_signatures
 }, {});
 
+#[cfg(taproot)]
+impl_writeable_msg!(CommitmentSigned, {
+       channel_id,
+       signature,
+       htlc_signatures
+}, {
+       (2, partial_signature_with_nonce, option)
+});
+
 impl_writeable!(DecodedOnionErrorPacket, {
        hmac,
        failuremsg,
        pad
 });
 
+#[cfg(not(taproot))]
 impl_writeable_msg!(FundingCreated, {
        temporary_channel_id,
        funding_txid,
        funding_output_index,
        signature
 }, {});
+#[cfg(taproot)]
+impl_writeable_msg!(FundingCreated, {
+       temporary_channel_id,
+       funding_txid,
+       funding_output_index,
+       signature
+}, {
+       (2, partial_signature_with_nonce, option),
+       (4, next_local_nonce, option)
+});
 
+#[cfg(not(taproot))]
 impl_writeable_msg!(FundingSigned, {
        channel_id,
        signature
 }, {});
 
+#[cfg(taproot)]
+impl_writeable_msg!(FundingSigned, {
+       channel_id,
+       signature
+}, {
+       (2, partial_signature_with_nonce, option)
+});
+
 impl_writeable_msg!(ChannelReady, {
        channel_id,
        next_per_commitment_point,
@@ -1446,12 +1522,22 @@ impl_writeable_msg!(OpenChannel, {
        (1, channel_type, option),
 });
 
+#[cfg(not(taproot))]
 impl_writeable_msg!(RevokeAndACK, {
        channel_id,
        per_commitment_secret,
        next_per_commitment_point
 }, {});
 
+#[cfg(taproot)]
+impl_writeable_msg!(RevokeAndACK, {
+       channel_id,
+       per_commitment_secret,
+       next_per_commitment_point
+}, {
+       (4, next_local_nonce, option)
+});
+
 impl_writeable_msg!(Shutdown, {
        channel_id,
        scriptpubkey
@@ -1576,11 +1662,12 @@ impl Writeable for OnionHopData {
                                        (6, short_channel_id, required)
                                });
                        },
-                       OnionHopDataFormat::FinalNode { ref payment_data, ref keysend_preimage } => {
+                       OnionHopDataFormat::FinalNode { ref payment_data, ref payment_metadata, ref keysend_preimage } => {
                                _encode_varint_length_prefixed_tlv!(w, {
                                        (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
                                        (4, HighZeroBytesDroppedBigSize(self.outgoing_cltv_value), required),
                                        (8, payment_data, option),
+                                       (16, payment_metadata.as_ref().map(|m| WithoutLength(m)), option),
                                        (5482373484, keysend_preimage, option)
                                });
                        },
@@ -1595,29 +1682,33 @@ impl Readable for OnionHopData {
                let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
                let mut short_id: Option<u64> = None;
                let mut payment_data: Option<FinalOnionHopData> = None;
+               let mut payment_metadata: Option<WithoutLength<Vec<u8>>> = None;
                let mut keysend_preimage: Option<PaymentPreimage> = None;
                read_tlv_fields!(r, {
                        (2, amt, required),
                        (4, cltv_value, required),
                        (6, short_id, option),
                        (8, payment_data, option),
+                       (16, payment_metadata, option),
                        // See https://github.com/lightning/blips/blob/master/blip-0003.md
                        (5482373484, keysend_preimage, option)
                });
 
                let format = if let Some(short_channel_id) = short_id {
                        if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
+                       if payment_metadata.is_some() { return Err(DecodeError::InvalidValue); }
                        OnionHopDataFormat::NonFinalNode {
                                short_channel_id,
                        }
                } else {
-                       if let &Some(ref data) = &payment_data {
+                       if let Some(data) = &payment_data {
                                if data.total_msat > MAX_VALUE_MSAT {
                                        return Err(DecodeError::InvalidValue);
                                }
                        }
                        OnionHopDataFormat::FinalNode {
                                payment_data,
+                               payment_metadata: payment_metadata.map(|w| w.0),
                                keysend_preimage,
                        }
                };
@@ -1846,7 +1937,7 @@ impl Readable for UnsignedNodeAnnouncement {
                let node_id: NodeId = Readable::read(r)?;
                let mut rgb = [0; 3];
                r.read_exact(&mut rgb)?;
-               let alias: [u8; 32] = Readable::read(r)?;
+               let alias: NodeAlias = Readable::read(r)?;
 
                let addr_len: u16 = Readable::read(r)?;
                let mut addresses: Vec<NetAddress> = Vec::new();
@@ -2053,7 +2144,7 @@ mod tests {
        use crate::ln::features::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, NodeFeatures};
        use crate::ln::msgs;
        use crate::ln::msgs::{FinalOnionHopData, OptionalField, OnionErrorPacket, OnionHopDataFormat};
-       use crate::routing::gossip::NodeId;
+       use crate::routing::gossip::{NodeAlias, NodeId};
        use crate::util::ser::{Writeable, Readable, Hostname};
 
        use bitcoin::hashes::hex::FromHex;
@@ -2248,7 +2339,7 @@ mod tests {
                        timestamp: 20190119,
                        node_id: NodeId::from_pubkey(&pubkey_1),
                        rgb: [32; 3],
-                       alias: [16;32],
+                       alias: NodeAlias([16;32]),
                        addresses,
                        excess_address_data: if excess_address_data { vec![33, 108, 40, 11, 83, 149, 162, 84, 110, 126, 75, 38, 99, 224, 79, 129, 22, 34, 241, 90, 79, 146, 232, 58, 162, 233, 43, 162, 165, 115, 193, 57, 20, 44, 84, 174, 99, 7, 42, 30, 193, 238, 125, 192, 192, 75, 222, 92, 132, 120, 6, 23, 42, 160, 92, 146, 194, 42, 232, 227, 8, 209, 210, 105] } else { Vec::new() },
                        excess_data: if excess_data { vec![59, 18, 204, 25, 92, 224, 162, 209, 189, 166, 168, 139, 239, 161, 159, 160, 127, 81, 202, 167, 92, 232, 56, 55, 242, 137, 101, 96, 11, 138, 172, 171, 8, 85, 255, 176, 231, 65, 236, 95, 124, 65, 66, 30, 152, 41, 169, 212, 134, 17, 200, 200, 49, 247, 27, 229, 234, 115, 230, 101, 148, 151, 127, 253] } else { Vec::new() },
@@ -2444,6 +2535,8 @@ mod tests {
                        first_per_commitment_point: pubkey_6,
                        shutdown_scriptpubkey: if shutdown { OptionalField::Present(Address::p2pkh(&::bitcoin::PublicKey{compressed: true, inner: pubkey_1}, Network::Testnet).script_pubkey()) } else { OptionalField::Absent },
                        channel_type: None,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                };
                let encoded_value = accept_channel.encode();
                let mut target_value = hex::decode("020202020202020202020202020202020202020202020202020202020202020212345678901234562334032891223698321446687011447600083a840000034d000c89d4c0bcc0bc031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f024d4b6cd1361032ca9bd2aeb9d900aa4d45d9ead80ac9423374c451a7254d076602531fe6068134503d2723133227c867ac8fa6c83c537e9a44c3c5bdbdcb1fe33703462779ad4aad39514614751a71085f2f10e1c7a593e4e030efb5b8721ce55b0b0362c0a046dacce86ddd0343c6d3c7c79c2208ba0d9c9cf24a6d046d21d21f90f703f006a18d5653c4edf5391ff23a61f03ff83d237e880ee61187fa9f379a028e0a").unwrap();
@@ -2469,6 +2562,10 @@ mod tests {
                        funding_txid: Txid::from_hex("c2d4449afa8d26140898dd54d3390b057ba2a5afcf03ba29d7dc0d8b9ffe966e").unwrap(),
                        funding_output_index: 255,
                        signature: sig_1,
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                };
                let encoded_value = funding_created.encode();
                let target_value = hex::decode("02020202020202020202020202020202020202020202020202020202020202026e96fe9f8b0ddcd729ba03cfafa5a27b050b39d354dd980814268dfa9a44d4c200ffd977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
@@ -2483,6 +2580,8 @@ mod tests {
                let funding_signed = msgs::FundingSigned {
                        channel_id: [2; 32],
                        signature: sig_1,
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
                };
                let encoded_value = funding_signed.encode();
                let target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
@@ -2646,6 +2745,8 @@ mod tests {
                        channel_id: [2; 32],
                        signature: sig_1,
                        htlc_signatures: if htlcs { vec![sig_2, sig_3, sig_4] } else { Vec::new() },
+                       #[cfg(taproot)]
+                       partial_signature_with_nonce: None,
                };
                let encoded_value = commitment_signed.encode();
                let mut target_value = hex::decode("0202020202020202020202020202020202020202020202020202020202020202d977cb9b53d93a6ff64bb5f1e158b4094b66e798fb12911168a3ccdf80a83096340a6a95da0ae8d9f776528eecdbb747eb6b545495a4319ed5378e35b21e073a").unwrap();
@@ -2671,6 +2772,8 @@ mod tests {
                        channel_id: [2; 32],
                        per_commitment_secret: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
                        next_per_commitment_point: pubkey_1,
+                       #[cfg(taproot)]
+                       next_local_nonce: None,
                };
                let encoded_value = raa.encode();
                let target_value = hex::decode("02020202020202020202020202020202020202020202020202020202020202020101010101010101010101010101010101010101010101010101010101010101031b84c5567b126440995d3ed5aaba0565d71e1834604819ff9c17f5e9d5dd078f").unwrap();
@@ -2783,6 +2886,7 @@ mod tests {
                let mut msg = msgs::OnionHopData {
                        format: OnionHopDataFormat::FinalNode {
                                payment_data: None,
+                               payment_metadata: None,
                                keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
@@ -2806,6 +2910,7 @@ mod tests {
                                        payment_secret: expected_payment_secret,
                                        total_msat: 0x1badca1f
                                }),
+                               payment_metadata: None,
                                keysend_preimage: None,
                        },
                        amt_to_forward: 0x0badf00d01020304,
@@ -2820,6 +2925,7 @@ mod tests {
                                payment_secret,
                                total_msat: 0x1badca1f
                        }),
+                       payment_metadata: None,
                        keysend_preimage: None,
                } = msg.format {
                        assert_eq!(payment_secret, expected_payment_secret);
index d71a5b11c089740e0f9e7ed1394eb740ae1b5006..d70016cf4ac1e854998a86e48513665238665f21 100644 (file)
 
 use crate::chain::channelmonitor::{CLTV_CLAIM_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
 use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient};
-use crate::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure};
+use crate::events::{Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
 use crate::ln::{PaymentHash, PaymentSecret};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{HTLCForwardInfo, FailureCode, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId};
+use crate::ln::channelmanager::{HTLCForwardInfo, FailureCode, CLTV_FAR_FAR_AWAY, DISABLE_GOSSIP_TICKS, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId, RecipientOnionFields};
 use crate::ln::onion_utils;
 use crate::routing::gossip::{NetworkUpdate, RoutingFees};
 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop};
@@ -82,7 +82,8 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
 
        // 0 ~~> 2 send payment
        let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
-       nodes[0].node.send_payment(&route, *payment_hash, &Some(*payment_secret), payment_id).unwrap();
+       nodes[0].node.send_payment_with_route(&route, *payment_hash,
+               RecipientOnionFields::secret_only(*payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        // temper update_add (0 => 1)
@@ -212,9 +213,14 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
                panic!("Unexpected event");
        }
        match events[1] {
-               Event::PaymentFailed { payment_hash: ev_payment_hash, payment_id: ev_payment_id } => {
+               Event::PaymentFailed { payment_hash: ev_payment_hash, payment_id: ev_payment_id, reason: ref ev_reason } => {
                        assert_eq!(*payment_hash, ev_payment_hash);
                        assert_eq!(payment_id, ev_payment_id);
+                       assert_eq!(if expected_retryable {
+                               PaymentFailureReason::RetriesExhausted
+                       } else {
+                               PaymentFailureReason::RecipientRejected
+                       }, ev_reason.unwrap());
                }
                _ => panic!("Unexpected second event"),
        }
@@ -280,7 +286,8 @@ fn test_fee_failures() {
 
        // positive case
        let (route, payment_hash_success, payment_preimage_success, payment_secret_success) = get_route_and_payment_hash!(nodes[0], nodes[2], 40_000);
-       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash_success,
+               RecipientOnionFields::secret_only(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
@@ -301,7 +308,8 @@ fn test_fee_failures() {
        }
 
        let (payment_preimage_success, payment_hash_success, payment_secret_success) = get_payment_preimage_hash!(nodes[2]);
-       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash_success,
+               RecipientOnionFields::secret_only(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
@@ -342,7 +350,8 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let cur_height = nodes[0].best_block_info().1 + 1;
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
-               let (mut onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, cur_height, &None).unwrap();
+               let (mut onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(
+                       &route.paths[0], 40000, RecipientOnionFields::spontaneous_empty(), cur_height, &None).unwrap();
                let mut new_payloads = Vec::new();
                for payload in onion_payloads.drain(..) {
                        new_payloads.push(BogusOnionHopData::new(payload));
@@ -359,7 +368,8 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let cur_height = nodes[0].best_block_info().1 + 1;
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
-               let (mut onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, cur_height, &None).unwrap();
+               let (mut onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(
+                       &route.paths[0], 40000, RecipientOnionFields::spontaneous_empty(), cur_height, &None).unwrap();
                let mut new_payloads = Vec::new();
                for payload in onion_payloads.drain(..) {
                        new_payloads.push(BogusOnionHopData::new(payload));
@@ -381,7 +391,7 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[0].shared_secret.as_ref(), NODE|2, &[0;0]);
-       }, ||{}, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: false}), Some(route.paths[0][0].short_channel_id));
+       }, ||{}, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[0].pubkey, is_permanent: false}), Some(route.paths[0].hops[0].short_channel_id));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("temporary_node_failure", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -391,7 +401,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[1].shared_secret.as_ref(), NODE|2, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: false}), Some(route.paths[0][1].short_channel_id));
+       }, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[1].pubkey, is_permanent: false}), Some(route.paths[0].hops[1].short_channel_id));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        // intermediate node failure
@@ -401,7 +411,7 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[0].shared_secret.as_ref(), PERM|NODE|2, &[0;0]);
-       }, ||{}, true, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}), Some(route.paths[0][0].short_channel_id));
+       }, ||{}, true, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[0].pubkey, is_permanent: true}), Some(route.paths[0].hops[0].short_channel_id));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("permanent_node_failure", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -410,7 +420,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[1].shared_secret.as_ref(), PERM|NODE|2, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, false, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}), Some(route.paths[0][1].short_channel_id));
+       }, false, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[1].pubkey, is_permanent: true}), Some(route.paths[0].hops[1].short_channel_id));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        // intermediate node failure
@@ -422,7 +432,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[0].shared_secret.as_ref(), PERM|NODE|3, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, true, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}), Some(route.paths[0][0].short_channel_id));
+       }, true, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[0].pubkey, is_permanent: true}), Some(route.paths[0].hops[0].short_channel_id));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("required_node_feature_missing", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -431,7 +441,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(onion_keys[1].shared_secret.as_ref(), PERM|NODE|3, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, false, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}), Some(route.paths[0][1].short_channel_id));
+       }, false, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[1].pubkey, is_permanent: true}), Some(route.paths[0].hops[1].short_channel_id));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        // Our immediate peer sent UpdateFailMalformedHTLC because it couldn't understand the onion in
@@ -492,8 +502,8 @@ fn test_onion_failure() {
        }, ||{}, true, Some(PERM|9), Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent: true}), Some(short_channel_id));
 
        let mut bogus_route = route.clone();
-       bogus_route.paths[0][1].short_channel_id -= 1;
-       let short_channel_id = bogus_route.paths[0][1].short_channel_id;
+       bogus_route.paths[0].hops[1].short_channel_id -= 1;
+       let short_channel_id = bogus_route.paths[0].hops[1].short_channel_id;
        run_onion_failure_test("unknown_next_peer", 0, &nodes, &bogus_route, &payment_hash, &payment_secret, |_| {}, ||{}, true, Some(PERM|10),
          Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent:true}), Some(short_channel_id));
 
@@ -502,8 +512,8 @@ fn test_onion_failure() {
                .unwrap().lock().unwrap().channel_by_id.get(&channels[1].2).unwrap()
                .get_counterparty_htlc_minimum_msat() - 1;
        let mut bogus_route = route.clone();
-       let route_len = bogus_route.paths[0].len();
-       bogus_route.paths[0][route_len-1].fee_msat = amt_to_forward;
+       let route_len = bogus_route.paths[0].hops.len();
+       bogus_route.paths[0].hops[route_len-1].fee_msat = amt_to_forward;
        run_onion_failure_test("amount_below_minimum", 0, &nodes, &bogus_route, &payment_hash, &payment_secret, |_| {}, ||{}, true, Some(UPDATE|11), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy(short_channel_id)}), Some(short_channel_id));
 
        // Clear pending payments so that the following positive test has the correct payment hash.
@@ -512,7 +522,7 @@ fn test_onion_failure() {
        }
 
        // Test a positive test-case with one extra msat, meeting the minimum.
-       bogus_route.paths[0][route_len-1].fee_msat = amt_to_forward + 1;
+       bogus_route.paths[0].hops[route_len-1].fee_msat = amt_to_forward + 1;
        let preimage = send_along_route(&nodes[0], bogus_route, &[&nodes[1], &nodes[2]], amt_to_forward+1).0;
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], preimage);
 
@@ -577,6 +587,15 @@ fn test_onion_failure() {
                // disconnect event to the channel between nodes[1] ~ nodes[2]
                nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id());
                nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
+       }, true, Some(UPDATE|7), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy(short_channel_id)}), Some(short_channel_id));
+       run_onion_failure_test("channel_disabled", 0, &nodes, &route, &payment_hash, &payment_secret, |_| {}, || {
+               // disconnect event to the channel between nodes[1] ~ nodes[2]
+               for _ in 0..DISABLE_GOSSIP_TICKS + 1 {
+                       nodes[1].node.timer_tick_occurred();
+                       nodes[2].node.timer_tick_occurred();
+               }
+               nodes[1].node.get_and_clear_pending_msg_events();
+               nodes[2].node.get_and_clear_pending_msg_events();
        }, true, Some(UPDATE|20), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy(short_channel_id)}), Some(short_channel_id));
        reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 
@@ -584,13 +603,14 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let mut route = route.clone();
                let height = nodes[2].best_block_info().1;
-               route.paths[0][1].cltv_expiry_delta += CLTV_FAR_FAR_AWAY + route.paths[0][0].cltv_expiry_delta + 1;
+               route.paths[0].hops[1].cltv_expiry_delta += CLTV_FAR_FAR_AWAY + route.paths[0].hops[0].cltv_expiry_delta + 1;
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
-               let (onion_payloads, _, htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, height, &None).unwrap();
+               let (onion_payloads, _, htlc_cltv) = onion_utils::build_onion_payloads(
+                       &route.paths[0], 40000, RecipientOnionFields::spontaneous_empty(), height, &None).unwrap();
                let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
                msg.cltv_expiry = htlc_cltv;
                msg.onion_routing_packet = onion_packet;
-       }, ||{}, true, Some(21), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}), Some(route.paths[0][0].short_channel_id));
+       }, ||{}, true, Some(21), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0].hops[0].pubkey, is_permanent: true}), Some(route.paths[0].hops[0].short_channel_id));
 
        run_onion_failure_test_with_fail_intercept("mpp_timeout", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
                // Tamper returning error message
@@ -613,7 +633,7 @@ fn test_overshoot_final_cltv() {
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 40000);
 
        let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
 
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -844,14 +864,15 @@ fn test_always_create_tlv_format_onion_payloads() {
                .with_features(InvoiceFeatures::empty());
        let (route, _payment_hash, _payment_preimage, _payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 40000, TEST_FINAL_CLTV);
 
-       let hops = &route.paths[0];
+       let hops = &route.paths[0].hops;
        // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[0].node_features.supports_variable_length_onion());
        // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[1].node_features.supports_variable_length_onion());
 
        let cur_height = nodes[0].best_block_info().1 + 1;
-       let (onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, cur_height, &None).unwrap();
+       let (onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(
+               &route.paths[0], 40000, RecipientOnionFields::spontaneous_empty(), cur_height, &None).unwrap();
 
        match onion_payloads[0].format {
                msgs::OnionHopDataFormat::NonFinalNode {..} => {},
@@ -880,7 +901,8 @@ fn do_test_fail_htlc_backwards_with_reason(failure_code: FailureCode) {
 
        let payment_amount = 100_000;
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], payment_amount);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -991,7 +1013,8 @@ fn test_phantom_onion_hmac_failure() {
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1052,7 +1075,8 @@ fn test_phantom_invalid_onion_payload() {
        // We'll use the session priv later when constructing an invalid onion packet.
        let session_priv = [3; 32];
        *nodes[0].keys_manager.override_random_bytes.lock().unwrap() = Some(session_priv);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1074,7 +1098,9 @@ fn test_phantom_invalid_onion_payload() {
                                        let height = nodes[0].best_block_info().1;
                                        let session_priv = SecretKey::from_slice(&session_priv).unwrap();
                                        let mut onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
-                                       let (mut onion_payloads, _, _) = onion_utils::build_onion_payloads(&route.paths[0], msgs::MAX_VALUE_MSAT + 1, &Some(payment_secret), height + 1, &None).unwrap();
+                                       let (mut onion_payloads, _, _) = onion_utils::build_onion_payloads(
+                                               &route.paths[0], msgs::MAX_VALUE_MSAT + 1,
+                                               RecipientOnionFields::secret_only(payment_secret), height + 1, &None).unwrap();
                                        // We only want to construct the onion packet for the last hop, not the entire route, so
                                        // remove the first hop's payload and its keys.
                                        onion_keys.remove(0);
@@ -1123,7 +1149,8 @@ fn test_phantom_final_incorrect_cltv_expiry() {
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1179,10 +1206,11 @@ fn test_phantom_failure_too_low_cltv() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Modify the route to have a too-low cltv.
-       route.paths[0][1].cltv_expiry_delta = 5;
+       route.paths[0].hops[1].cltv_expiry_delta = 5;
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1229,7 +1257,8 @@ fn test_phantom_failure_modified_cltv() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1270,7 +1299,8 @@ fn test_phantom_failure_expires_too_soon() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1308,7 +1338,8 @@ fn test_phantom_failure_too_low_recv_amt() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, bad_recv_amt_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1358,7 +1389,8 @@ fn test_phantom_dust_exposure_failure() {
        let (mut route, _) = get_phantom_route!(nodes, max_dust_exposure + 1, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1401,7 +1433,8 @@ fn test_phantom_failure_reject_payment() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_amt_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1413,7 +1446,7 @@ fn test_phantom_failure_reject_payment() {
        nodes[1].node.process_pending_htlc_forwards();
        expect_pending_htlcs_forwardable_ignore!(nodes[1]);
        nodes[1].node.process_pending_htlc_forwards();
-       expect_payment_claimable!(nodes[1], payment_hash, payment_secret, recv_amt_msat, None, route.paths[0].last().unwrap().pubkey);
+       expect_payment_claimable!(nodes[1], payment_hash, payment_secret, recv_amt_msat, None, route.paths[0].hops.last().unwrap().pubkey);
        nodes[1].node.fail_htlc_backwards(&payment_hash);
        expect_pending_htlcs_forwardable_and_htlc_handling_failed_ignore!(nodes[1], vec![HTLCDestination::FailedPayment { payment_hash }]);
        nodes[1].node.process_pending_htlc_forwards();
index 2916829f259ec58ee110b95fdf14fb16a702a401..ebcf83bd90dc2d0db68c5300e50cf2f6b06dd718 100644 (file)
@@ -7,12 +7,12 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-use crate::ln::channelmanager::HTLCSource;
+use crate::ln::{PaymentHash, PaymentPreimage};
+use crate::ln::channelmanager::{HTLCSource, RecipientOnionFields};
 use crate::ln::msgs;
 use crate::ln::wire::Encode;
 use crate::routing::gossip::NetworkUpdate;
-use crate::routing::router::RouteHop;
+use crate::routing::router::{Path, RouteHop};
 use crate::util::chacha20::{ChaCha20, ChaChaReader};
 use crate::util::errors::{self, APIError};
 use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, LengthCalculatingWriter};
@@ -128,10 +128,10 @@ pub(super) fn construct_onion_keys_callback<T: secp256k1::Signing, FType: FnMut(
 }
 
 // can only fail if an intermediary hop has an invalid public key or session_priv is invalid
-pub(super) fn construct_onion_keys<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, path: &Vec<RouteHop>, session_priv: &SecretKey) -> Result<Vec<OnionKeys>, secp256k1::Error> {
-       let mut res = Vec::with_capacity(path.len());
+pub(super) fn construct_onion_keys<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T>, path: &Path, session_priv: &SecretKey) -> Result<Vec<OnionKeys>, secp256k1::Error> {
+       let mut res = Vec::with_capacity(path.hops.len());
 
-       construct_onion_keys_callback(secp_ctx, path, session_priv, |shared_secret, _blinding_factor, ephemeral_pubkey, _, _| {
+       construct_onion_keys_callback(secp_ctx, &path.hops, session_priv, |shared_secret, _blinding_factor, ephemeral_pubkey, _, _| {
                let (rho, mu) = gen_rho_mu_from_shared_secret(shared_secret.as_ref());
 
                res.push(OnionKeys {
@@ -149,13 +149,13 @@ pub(super) fn construct_onion_keys<T: secp256k1::Signing>(secp_ctx: &Secp256k1<T
 }
 
 /// returns the hop data, as well as the first-hop value_msat and CLTV value we should send.
-pub(super) fn build_onion_payloads(path: &Vec<RouteHop>, total_msat: u64, payment_secret_option: &Option<PaymentSecret>, starting_htlc_offset: u32, keysend_preimage: &Option<PaymentPreimage>) -> Result<(Vec<msgs::OnionHopData>, u64, u32), APIError> {
+pub(super) fn build_onion_payloads(path: &Path, total_msat: u64, mut recipient_onion: RecipientOnionFields, starting_htlc_offset: u32, keysend_preimage: &Option<PaymentPreimage>) -> Result<(Vec<msgs::OnionHopData>, u64, u32), APIError> {
        let mut cur_value_msat = 0u64;
        let mut cur_cltv = starting_htlc_offset;
        let mut last_short_channel_id = 0;
-       let mut res: Vec<msgs::OnionHopData> = Vec::with_capacity(path.len());
+       let mut res: Vec<msgs::OnionHopData> = Vec::with_capacity(path.hops.len());
 
-       for (idx, hop) in path.iter().rev().enumerate() {
+       for (idx, hop) in path.hops.iter().rev().enumerate() {
                // First hop gets special values so that it can check, on receipt, that everything is
                // exactly as it should be (and the next hop isn't trying to probe to find out if we're
                // the intended recipient).
@@ -164,12 +164,13 @@ pub(super) fn build_onion_payloads(path: &Vec<RouteHop>, total_msat: u64, paymen
                res.insert(0, msgs::OnionHopData {
                        format: if idx == 0 {
                                msgs::OnionHopDataFormat::FinalNode {
-                                       payment_data: if let &Some(ref payment_secret) = payment_secret_option {
+                                       payment_data: if let Some(secret) = recipient_onion.payment_secret.take() {
                                                Some(msgs::FinalOnionHopData {
-                                                       payment_secret: payment_secret.clone(),
+                                                       payment_secret: secret,
                                                        total_msat,
                                                })
                                        } else { None },
+                                       payment_metadata: recipient_onion.payment_metadata.take(),
                                        keysend_preimage: *keysend_preimage,
                                }
                        } else {
@@ -402,7 +403,7 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                let mut is_from_final_node = false;
 
                // Handle packed channel/node updates for passing back for the route handler
-               construct_onion_keys_callback(secp_ctx, path, session_priv, |shared_secret, _, _, route_hop, route_hop_idx| {
+               construct_onion_keys_callback(secp_ctx, &path.hops, session_priv, |shared_secret, _, _, route_hop, route_hop_idx| {
                        if res.is_some() { return; }
 
                        let amt_to_forward = htlc_msat - route_hop.fee_msat;
@@ -418,8 +419,8 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
 
                        // The failing hop includes either the inbound channel to the recipient or the outbound
                        // channel from the current hop (i.e., the next hop's inbound channel).
-                       is_from_final_node = route_hop_idx + 1 == path.len();
-                       let failing_route_hop = if is_from_final_node { route_hop } else { &path[route_hop_idx + 1] };
+                       is_from_final_node = route_hop_idx + 1 == path.hops.len();
+                       let failing_route_hop = if is_from_final_node { route_hop } else { &path.hops[route_hop_idx + 1] };
 
                        if let Ok(err_packet) = msgs::DecodedOnionErrorPacket::read(&mut Cursor::new(&packet_decrypted)) {
                                let um = gen_um_from_shared_secret(shared_secret.as_ref());
@@ -492,21 +493,28 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                                        } else {
                                                                                log_trace!(logger, "Failure provided features a channel update without type prefix. Deprecated, but allowing for now.");
                                                                        }
-                                                                       if let Ok(chan_update) = msgs::ChannelUpdate::read(&mut Cursor::new(&update_slice)) {
+                                                                       let update_opt = msgs::ChannelUpdate::read(&mut Cursor::new(&update_slice));
+                                                                       if update_opt.is_ok() || update_slice.is_empty() {
                                                                                // if channel_update should NOT have caused the failure:
                                                                                // MAY treat the channel_update as invalid.
                                                                                let is_chan_update_invalid = match error_code & 0xff {
                                                                                        7 => false,
-                                                                                       11 => amt_to_forward > chan_update.contents.htlc_minimum_msat,
-                                                                                       12 => amt_to_forward
-                                                                                               .checked_mul(chan_update.contents.fee_proportional_millionths as u64)
+                                                                                       11 => update_opt.is_ok() &&
+                                                                                               amt_to_forward >
+                                                                                                       update_opt.as_ref().unwrap().contents.htlc_minimum_msat,
+                                                                                       12 => update_opt.is_ok() && amt_to_forward
+                                                                                               .checked_mul(update_opt.as_ref().unwrap()
+                                                                                                       .contents.fee_proportional_millionths as u64)
                                                                                                .map(|prop_fee| prop_fee / 1_000_000)
-                                                                                               .and_then(|prop_fee| prop_fee.checked_add(chan_update.contents.fee_base_msat as u64))
+                                                                                               .and_then(|prop_fee| prop_fee.checked_add(
+                                                                                                       update_opt.as_ref().unwrap().contents.fee_base_msat as u64))
                                                                                                .map(|fee_msats| route_hop.fee_msat >= fee_msats)
                                                                                                .unwrap_or(false),
-                                                                                       13 => route_hop.cltv_expiry_delta as u16 >= chan_update.contents.cltv_expiry_delta,
+                                                                                       13 => update_opt.is_ok() &&
+                                                                                               route_hop.cltv_expiry_delta as u16 >=
+                                                                                                       update_opt.as_ref().unwrap().contents.cltv_expiry_delta,
                                                                                        14 => false, // expiry_too_soon; always valid?
-                                                                                       20 => chan_update.contents.flags & 2 == 0,
+                                                                                       20 => update_opt.as_ref().unwrap().contents.flags & 2 == 0,
                                                                                        _ => false, // unknown error code; take channel_update as valid
                                                                                };
                                                                                if is_chan_update_invalid {
@@ -517,17 +525,31 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                                                                is_permanent: true,
                                                                                        });
                                                                                } else {
-                                                                                       // Make sure the ChannelUpdate contains the expected
-                                                                                       // short channel id.
-                                                                                       if failing_route_hop.short_channel_id == chan_update.contents.short_channel_id {
-                                                                                               short_channel_id = Some(failing_route_hop.short_channel_id);
+                                                                                       if let Ok(chan_update) = update_opt {
+                                                                                               // Make sure the ChannelUpdate contains the expected
+                                                                                               // short channel id.
+                                                                                               if failing_route_hop.short_channel_id == chan_update.contents.short_channel_id {
+                                                                                                       short_channel_id = Some(failing_route_hop.short_channel_id);
+                                                                                               } else {
+                                                                                                       log_info!(logger, "Node provided a channel_update for which it was not authoritative, ignoring.");
+                                                                                               }
+                                                                                               network_update = Some(NetworkUpdate::ChannelUpdateMessage {
+                                                                                                       msg: chan_update,
+                                                                                               })
                                                                                        } else {
-                                                                                               log_info!(logger, "Node provided a channel_update for which it was not authoritative, ignoring.");
+                                                                                               network_update = Some(NetworkUpdate::ChannelFailure {
+                                                                                                       short_channel_id: route_hop.short_channel_id,
+                                                                                                       is_permanent: false,
+                                                                                               });
                                                                                        }
-                                                                                       network_update = Some(NetworkUpdate::ChannelUpdateMessage {
-                                                                                               msg: chan_update,
-                                                                                       })
                                                                                };
+                                                                       } else {
+                                                                               // If the channel_update had a non-zero length (i.e. was
+                                                                               // present) but we couldn't read it, treat it as a total
+                                                                               // node failure.
+                                                                               log_info!(logger,
+                                                                                       "Failed to read a channel_update of len {} in an onion",
+                                                                                       update_slice.len());
                                                                        }
                                                                }
                                                        }
@@ -725,7 +747,7 @@ impl HTLCFailReason {
                                // generally ignores its view of our own channels as we provide them via
                                // ChannelDetails.
                                if let &HTLCSource::OutboundRoute { ref path, .. } = htlc_source {
-                                       (None, Some(path.first().unwrap().short_channel_id), true, Some(*failure_code), Some(data.clone()))
+                                       (None, Some(path.hops[0].short_channel_id), true, Some(*failure_code), Some(data.clone()))
                                } else { unreachable!(); }
                        }
                }
@@ -882,7 +904,7 @@ mod tests {
        use crate::prelude::*;
        use crate::ln::PaymentHash;
        use crate::ln::features::{ChannelFeatures, NodeFeatures};
-       use crate::routing::router::{Route, RouteHop};
+       use crate::routing::router::{Path, Route, RouteHop};
        use crate::ln::msgs;
        use crate::util::ser::{Writeable, Writer, VecWriter};
 
@@ -902,7 +924,7 @@ mod tests {
                let secp_ctx = Secp256k1::new();
 
                let route = Route {
-                       paths: vec![vec![
+                       paths: vec![Path { hops: vec![
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
@@ -928,12 +950,12 @@ mod tests {
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
                                                short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
-                       ]],
+                       ], blinded_tail: None }],
                        payment_params: None,
                };
 
                let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &get_test_session_key()).unwrap();
-               assert_eq!(onion_keys.len(), route.paths[0].len());
+               assert_eq!(onion_keys.len(), route.paths[0].hops.len());
                onion_keys
        }
 
index dd6227349f3f0968cd7595cd03fb9aa463716949..5270ed35d8835d6f240e8ae0fcd90b12e0851685 100644 (file)
@@ -14,11 +14,11 @@ use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::{self, Secp256k1, SecretKey};
 
 use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient};
-use crate::events;
+use crate::events::{self, PaymentFailureReason};
 use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use crate::ln::channelmanager::{ChannelDetails, HTLCSource, IDEMPOTENCY_TIMEOUT_TICKS, PaymentId};
 use crate::ln::onion_utils::HTLCFailReason;
-use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, RoutePath, Router};
+use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteParameters, Router};
 use crate::util::errors::APIError;
 use crate::util::logger::Logger;
 use crate::util::time::Time;
@@ -26,7 +26,6 @@ use crate::util::time::Time;
 use crate::util::time::tests::SinceEpoch;
 use crate::util::ser::ReadableArgs;
 
-use core::cmp;
 use core::fmt::{self, Display, Formatter};
 use core::ops::Deref;
 
@@ -46,6 +45,7 @@ pub(crate) enum PendingOutboundPayment {
                session_privs: HashSet<[u8; 32]>,
                payment_hash: PaymentHash,
                payment_secret: Option<PaymentSecret>,
+               payment_metadata: Option<Vec<u8>>,
                keysend_preimage: Option<PaymentPreimage>,
                pending_amt_msat: u64,
                /// Used to track the fee paid. Only present if the payment was serialized on 0.0.103+.
@@ -68,6 +68,8 @@ pub(crate) enum PendingOutboundPayment {
        Abandoned {
                session_privs: HashSet<[u8; 32]>,
                payment_hash: PaymentHash,
+               /// Will be `None` if the payment was serialized before 0.0.115.
+               reason: Option<PaymentFailureReason>,
        },
 }
 
@@ -145,25 +147,20 @@ impl PendingOutboundPayment {
                *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash, timer_ticks_without_htlcs: 0 };
        }
 
-       fn mark_abandoned(&mut self) -> Result<(), ()> {
-               let mut session_privs = HashSet::new();
-               let our_payment_hash;
-               core::mem::swap(&mut session_privs, match self {
-                       PendingOutboundPayment::Legacy { .. } |
-                       PendingOutboundPayment::Fulfilled { .. } =>
-                               return Err(()),
-                       PendingOutboundPayment::Retryable { session_privs, payment_hash, .. } |
-                       PendingOutboundPayment::Abandoned { session_privs, payment_hash, .. } => {
-                               our_payment_hash = *payment_hash;
-                               session_privs
-                       },
-               });
-               *self = PendingOutboundPayment::Abandoned { session_privs, payment_hash: our_payment_hash };
-               Ok(())
+       fn mark_abandoned(&mut self, reason: PaymentFailureReason) {
+               if let PendingOutboundPayment::Retryable { session_privs, payment_hash, .. } = self {
+                       let mut our_session_privs = HashSet::new();
+                       core::mem::swap(&mut our_session_privs, session_privs);
+                       *self = PendingOutboundPayment::Abandoned {
+                               session_privs: our_session_privs,
+                               payment_hash: *payment_hash,
+                               reason: Some(reason)
+                       };
+               }
        }
 
        /// panics if path is None and !self.is_fulfilled
-       fn remove(&mut self, session_priv: &[u8; 32], path: Option<&Vec<RouteHop>>) -> bool {
+       fn remove(&mut self, session_priv: &[u8; 32], path: Option<&Path>) -> bool {
                let remove_res = match self {
                        PendingOutboundPayment::Legacy { session_privs } |
                                PendingOutboundPayment::Retryable { session_privs, .. } |
@@ -175,17 +172,16 @@ impl PendingOutboundPayment {
                if remove_res {
                        if let PendingOutboundPayment::Retryable { ref mut pending_amt_msat, ref mut pending_fee_msat, .. } = self {
                                let path = path.expect("Fulfilling a payment should always come with a path");
-                               let path_last_hop = path.last().expect("Outbound payments must have had a valid path");
-                               *pending_amt_msat -= path_last_hop.fee_msat;
+                               *pending_amt_msat -= path.final_value_msat();
                                if let Some(fee_msat) = pending_fee_msat.as_mut() {
-                                       *fee_msat -= path.get_path_fees();
+                                       *fee_msat -= path.fee_msat();
                                }
                        }
                }
                remove_res
        }
 
-       pub(super) fn insert(&mut self, session_priv: [u8; 32], path: &Vec<RouteHop>) -> bool {
+       pub(super) fn insert(&mut self, session_priv: [u8; 32], path: &Path) -> bool {
                let insert_res = match self {
                        PendingOutboundPayment::Legacy { session_privs } |
                                PendingOutboundPayment::Retryable { session_privs, .. } => {
@@ -196,10 +192,9 @@ impl PendingOutboundPayment {
                };
                if insert_res {
                        if let PendingOutboundPayment::Retryable { ref mut pending_amt_msat, ref mut pending_fee_msat, .. } = self {
-                               let path_last_hop = path.last().expect("Outbound payments must have had a valid path");
-                               *pending_amt_msat += path_last_hop.fee_msat;
+                               *pending_amt_msat += path.final_value_msat();
                                if let Some(fee_msat) = pending_fee_msat.as_mut() {
-                                       *fee_msat += path.get_path_fees();
+                                       *fee_msat += path.fee_msat();
                                }
                        }
                }
@@ -310,10 +305,10 @@ impl<T: Time> Display for PaymentAttemptsUsingTime<T> {
        }
 }
 
-/// Indicates an immediate error on [`ChannelManager::send_payment_with_retry`]. Further errors
-/// may be surfaced later via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`].
+/// Indicates an immediate error on [`ChannelManager::send_payment`]. Further errors may be
+/// surfaced later via [`Event::PaymentPathFailed`] and [`Event::PaymentFailed`].
 ///
-/// [`ChannelManager::send_payment_with_retry`]: crate::ln::channelmanager::ChannelManager::send_payment_with_retry
+/// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
 /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed
 /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed
 #[derive(Clone, Debug)]
@@ -334,11 +329,11 @@ pub enum RetryableSendFailure {
        DuplicatePayment,
 }
 
-/// If a payment fails to send with [`ChannelManager::send_payment`], it can be in one of several
-/// states. This enum is returned as the Err() type describing which state the payment is in, see
-/// the description of individual enum states for more.
+/// If a payment fails to send with [`ChannelManager::send_payment_with_route`], it can be in one
+/// of several states. This enum is returned as the Err() type describing which state the payment
+/// is in, see the description of individual enum states for more.
 ///
-/// [`ChannelManager::send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
+/// [`ChannelManager::send_payment_with_route`]: crate::ln::channelmanager::ChannelManager::send_payment_with_route
 #[derive(Clone, Debug)]
 pub enum PaymentSendFailure {
        /// A parameter which was passed to send_payment was invalid, preventing us from attempting to
@@ -404,6 +399,76 @@ pub enum PaymentSendFailure {
        },
 }
 
+/// Information which is provided, encrypted, to the payment recipient when sending HTLCs.
+///
+/// This should generally be constructed with data communicated to us from the recipient (via a
+/// BOLT11 or BOLT12 invoice).
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct RecipientOnionFields {
+       /// The [`PaymentSecret`] is an arbitrary 32 bytes provided by the recipient for us to repeat
+       /// in the onion. It is unrelated to `payment_hash` (or [`PaymentPreimage`]) and exists to
+       /// authenticate the sender to the recipient and prevent payment-probing (deanonymization)
+       /// attacks.
+       ///
+       /// If you do not have one, the [`Route`] you pay over must not contain multiple paths as
+       /// multi-path payments require a recipient-provided secret.
+       ///
+       /// Note that for spontaneous payments most lightning nodes do not currently support MPP
+       /// receives, thus you should generally never be providing a secret here for spontaneous
+       /// payments.
+       pub payment_secret: Option<PaymentSecret>,
+       /// The payment metadata serves a similar purpose as [`Self::payment_secret`] but is of
+       /// arbitrary length. This gives recipients substantially more flexibility to receive
+       /// additional data.
+       ///
+       /// In LDK, while the [`Self::payment_secret`] is fixed based on an internal authentication
+       /// scheme to authenticate received payments against expected payments and invoices, this field
+       /// is not used in LDK for received payments, and can be used to store arbitrary data in
+       /// invoices which will be received with the payment.
+       ///
+       /// Note that this field was added to the lightning specification more recently than
+       /// [`Self::payment_secret`] and while nearly all lightning senders support secrets, metadata
+       /// may not be supported as universally.
+       pub payment_metadata: Option<Vec<u8>>,
+}
+
+impl_writeable_tlv_based!(RecipientOnionFields, {
+       (0, payment_secret, option),
+       (2, payment_metadata, option),
+});
+
+impl RecipientOnionFields {
+       /// Creates a [`RecipientOnionFields`] from only a [`PaymentSecret`]. This is the most common
+       /// set of onion fields for today's BOLT11 invoices - most nodes require a [`PaymentSecret`]
+       /// but do not require or provide any further data.
+       pub fn secret_only(payment_secret: PaymentSecret) -> Self {
+               Self { payment_secret: Some(payment_secret), payment_metadata: None }
+       }
+
+       /// Creates a new [`RecipientOnionFields`] with no fields. This generally does not create
+       /// payable HTLCs except for spontaneous payments, i.e. this should generally only be used for
+       /// calls to [`ChannelManager::send_spontaneous_payment`].
+       ///
+       /// [`ChannelManager::send_spontaneous_payment`]: super::channelmanager::ChannelManager::send_spontaneous_payment
+       pub fn spontaneous_empty() -> Self {
+               Self { payment_secret: None, payment_metadata: None }
+       }
+
+       /// When we have received some HTLC(s) towards an MPP payment, as we receive further HTLC(s) we
+       /// have to make sure that some fields match exactly across the parts. For those that aren't
+       /// required to match, if they don't match we should remove them so as to not expose data
+       /// that's dependent on the HTLC receive order to users.
+       ///
+       /// Here we implement this, first checking compatibility then mutating two objects and then
+       /// dropping any remaining non-matching fields from both.
+       pub(super) fn check_merge(&mut self, further_htlc_fields: &mut Self) -> Result<(), ()> {
+               if self.payment_secret != further_htlc_fields.payment_secret { return Err(()); }
+               if self.payment_metadata != further_htlc_fields.payment_metadata { return Err(()); }
+               // For custom TLVs we should just drop non-matching ones, but not reject the payment.
+               Ok(())
+       }
+}
+
 pub(super) struct OutboundPayments {
        pub(super) pending_outbound_payments: Mutex<HashMap<PaymentId, PendingOutboundPayment>>,
        pub(super) retry_lock: Mutex<()>,
@@ -418,7 +483,7 @@ impl OutboundPayments {
        }
 
        pub(super) fn send_payment<R: Deref, ES: Deref, NS: Deref, IH, SP, L: Deref>(
-               &self, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId,
+               &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId,
                retry_strategy: Retry, route_params: RouteParameters, router: &R,
                first_hops: Vec<ChannelDetails>, compute_inflight_htlcs: IH, entropy_source: &ES,
                node_signer: &NS, best_block_height: u32, logger: &L,
@@ -430,34 +495,34 @@ impl OutboundPayments {
                NS::Target: NodeSigner,
                L::Target: Logger,
                IH: Fn() -> InFlightHtlcs,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>,
        {
-               self.send_payment_internal(payment_id, payment_hash, payment_secret, None, retry_strategy,
+               self.send_payment_internal(payment_id, payment_hash, recipient_onion, None, retry_strategy,
                        route_params, router, first_hops, &compute_inflight_htlcs, entropy_source, node_signer,
                        best_block_height, logger, pending_events, &send_payment_along_path)
        }
 
        pub(super) fn send_payment_with_route<ES: Deref, NS: Deref, F>(
-               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                payment_id: PaymentId, entropy_source: &ES, node_signer: &NS, best_block_height: u32,
                send_payment_along_path: F
        ) -> Result<(), PaymentSendFailure>
        where
                ES::Target: EntropySource,
                NS::Target: NodeSigner,
-               F: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               F: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, None, route, None, None, entropy_source, best_block_height)?;
-               self.pay_route_internal(route, payment_hash, payment_secret, None, payment_id, None,
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, recipient_onion.clone(), payment_id, None, route, None, None, entropy_source, best_block_height)?;
+               self.pay_route_internal(route, payment_hash, recipient_onion, None, payment_id, None,
                        onion_session_privs, node_signer, best_block_height, &send_payment_along_path)
                        .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e })
        }
 
        pub(super) fn send_spontaneous_payment<R: Deref, ES: Deref, NS: Deref, IH, SP, L: Deref>(
-               &self, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId,
-               retry_strategy: Retry, route_params: RouteParameters, router: &R,
+               &self, payment_preimage: Option<PaymentPreimage>, recipient_onion: RecipientOnionFields,
+               payment_id: PaymentId, retry_strategy: Retry, route_params: RouteParameters, router: &R,
                first_hops: Vec<ChannelDetails>, inflight_htlcs: IH, entropy_source: &ES,
                node_signer: &NS, best_block_height: u32, logger: &L,
                pending_events: &Mutex<Vec<events::Event>>, send_payment_along_path: SP
@@ -468,34 +533,38 @@ impl OutboundPayments {
                NS::Target: NodeSigner,
                L::Target: Logger,
                IH: Fn() -> InFlightHtlcs,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>,
        {
                let preimage = payment_preimage
                        .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes()));
                let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
-               self.send_payment_internal(payment_id, payment_hash, &None, Some(preimage), retry_strategy,
-                       route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer,
-                       best_block_height, logger, pending_events, send_payment_along_path)
+               self.send_payment_internal(payment_id, payment_hash, recipient_onion, Some(preimage),
+                       retry_strategy, route_params, router, first_hops, inflight_htlcs, entropy_source,
+                       node_signer, best_block_height, logger, pending_events, send_payment_along_path)
                        .map(|()| payment_hash)
        }
 
        pub(super) fn send_spontaneous_payment_with_route<ES: Deref, NS: Deref, F>(
-               &self, route: &Route, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId,
-               entropy_source: &ES, node_signer: &NS, best_block_height: u32, send_payment_along_path: F
+               &self, route: &Route, payment_preimage: Option<PaymentPreimage>,
+               recipient_onion: RecipientOnionFields, payment_id: PaymentId, entropy_source: &ES,
+               node_signer: &NS, best_block_height: u32, send_payment_along_path: F
        ) -> Result<PaymentHash, PaymentSendFailure>
        where
                ES::Target: EntropySource,
                NS::Target: NodeSigner,
-               F: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               F: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                let preimage = payment_preimage
                        .unwrap_or_else(|| PaymentPreimage(entropy_source.get_secure_random_bytes()));
                let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, Some(preimage), &route, None, None, entropy_source, best_block_height)?;
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, recipient_onion.clone(),
+                       payment_id, Some(preimage), &route, None, None, entropy_source, best_block_height)?;
 
-               match self.pay_route_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path) {
+               match self.pay_route_internal(route, payment_hash, recipient_onion, Some(preimage),
+                       payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path
+               ) {
                        Ok(()) => Ok(payment_hash),
                        Err(e) => {
                                self.remove_outbound_if_all_failed(payment_id, &e);
@@ -513,7 +582,7 @@ impl OutboundPayments {
                R::Target: Router,
                ES::Target: EntropySource,
                NS::Target: NodeSigner,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>,
                IH: Fn() -> InFlightHtlcs,
                FH: Fn() -> Vec<ChannelDetails>,
@@ -546,10 +615,12 @@ impl OutboundPayments {
                outbounds.retain(|pmt_id, pmt| {
                        let mut retain = true;
                        if !pmt.is_auto_retryable_now() && pmt.remaining_parts() == 0 {
-                               if pmt.mark_abandoned().is_ok() {
+                               pmt.mark_abandoned(PaymentFailureReason::RetriesExhausted);
+                               if let PendingOutboundPayment::Abandoned { payment_hash, reason, .. } = pmt {
                                        pending_events.lock().unwrap().push(events::Event::PaymentFailed {
                                                payment_id: *pmt_id,
-                                               payment_hash: pmt.payment_hash().expect("PendingOutboundPayments::Retryable always has a payment hash set"),
+                                               payment_hash: *payment_hash,
+                                               reason: *reason,
                                        });
                                        retain = false;
                                }
@@ -570,7 +641,7 @@ impl OutboundPayments {
        /// [`Event::PaymentPathFailed`]: crate::events::Event::PaymentPathFailed
        /// [`Event::PaymentFailed`]: crate::events::Event::PaymentFailed
        fn send_payment_internal<R: Deref, NS: Deref, ES: Deref, IH, SP, L: Deref>(
-               &self, payment_id: PaymentId, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, payment_id: PaymentId, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                keysend_preimage: Option<PaymentPreimage>, retry_strategy: Retry, route_params: RouteParameters,
                router: &R, first_hops: Vec<ChannelDetails>, inflight_htlcs: IH, entropy_source: &ES,
                node_signer: &NS, best_block_height: u32, logger: &L,
@@ -582,7 +653,7 @@ impl OutboundPayments {
                NS::Target: NodeSigner,
                L::Target: Logger,
                IH: Fn() -> InFlightHtlcs,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                #[cfg(feature = "std")] {
@@ -597,12 +668,12 @@ impl OutboundPayments {
                        payment_hash, payment_id,
                ).map_err(|_| RetryableSendFailure::RouteNotFound)?;
 
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret,
-                       payment_id, keysend_preimage, &route, Some(retry_strategy),
+               let onion_session_privs = self.add_new_pending_payment(payment_hash,
+                       recipient_onion.clone(), payment_id, keysend_preimage, &route, Some(retry_strategy),
                        Some(route_params.payment_params.clone()), entropy_source, best_block_height)
                        .map_err(|_| RetryableSendFailure::DuplicatePayment)?;
 
-               let res = self.pay_route_internal(&route, payment_hash, payment_secret, None, payment_id, None,
+               let res = self.pay_route_internal(&route, payment_hash, recipient_onion, None, payment_id, None,
                        onion_session_privs, node_signer, best_block_height, &send_payment_along_path);
                log_info!(logger, "Result sending payment with id {}: {:?}", log_bytes!(payment_id.0), res);
                if let Err(e) = res {
@@ -623,13 +694,13 @@ impl OutboundPayments {
                NS::Target: NodeSigner,
                L::Target: Logger,
                IH: Fn() -> InFlightHtlcs,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                #[cfg(feature = "std")] {
                        if has_expired(&route_params) {
                                log_error!(logger, "Payment params expired on retry, abandoning payment {}", log_bytes!(payment_id.0));
-                               self.abandon_payment(payment_id, pending_events);
+                               self.abandon_payment(payment_id, PaymentFailureReason::PaymentExpired, pending_events);
                                return
                        }
                }
@@ -642,14 +713,14 @@ impl OutboundPayments {
                        Ok(route) => route,
                        Err(e) => {
                                log_error!(logger, "Failed to find a route on retry, abandoning payment {}: {:#?}", log_bytes!(payment_id.0), e);
-                               self.abandon_payment(payment_id, pending_events);
+                               self.abandon_payment(payment_id, PaymentFailureReason::RouteNotFound, pending_events);
                                return
                        }
                };
                for path in route.paths.iter() {
-                       if path.len() == 0 {
-                               log_error!(logger, "length-0 path in route");
-                               self.abandon_payment(payment_id, pending_events);
+                       if path.hops.len() == 0 {
+                               log_error!(logger, "Unusable path in route (path.hops.len() must be at least 1");
+                               self.abandon_payment(payment_id, PaymentFailureReason::UnexpectedError, pending_events);
                                return
                        }
                }
@@ -661,31 +732,38 @@ impl OutboundPayments {
                }
 
                macro_rules! abandon_with_entry {
-                       ($payment: expr) => {
-                               if $payment.get_mut().mark_abandoned().is_ok() && $payment.get().remaining_parts() == 0 {
-                                       pending_events.lock().unwrap().push(events::Event::PaymentFailed {
-                                               payment_id,
-                                               payment_hash,
-                                       });
-                                       $payment.remove();
+                       ($payment: expr, $reason: expr) => {
+                               $payment.get_mut().mark_abandoned($reason);
+                               if let PendingOutboundPayment::Abandoned { reason, .. } = $payment.get() {
+                                       if $payment.get().remaining_parts() == 0 {
+                                               pending_events.lock().unwrap().push(events::Event::PaymentFailed {
+                                                       payment_id,
+                                                       payment_hash,
+                                                       reason: *reason,
+                                               });
+                                               $payment.remove();
+                                       }
                                }
                        }
                }
-               let (total_msat, payment_secret, keysend_preimage) = {
+               let (total_msat, recipient_onion, keysend_preimage) = {
                        let mut outbounds = self.pending_outbound_payments.lock().unwrap();
                        match outbounds.entry(payment_id) {
                                hash_map::Entry::Occupied(mut payment) => {
                                        let res = match payment.get() {
                                                PendingOutboundPayment::Retryable {
-                                                       total_msat, keysend_preimage, payment_secret, pending_amt_msat, ..
+                                                       total_msat, keysend_preimage, payment_secret, payment_metadata, pending_amt_msat, ..
                                                } => {
-                                                       let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
+                                                       let retry_amt_msat = route.get_total_amount();
                                                        if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
                                                                log_error!(logger, "retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat);
-                                                               abandon_with_entry!(payment);
+                                                               abandon_with_entry!(payment, PaymentFailureReason::UnexpectedError);
                                                                return
                                                        }
-                                                       (*total_msat, *payment_secret, *keysend_preimage)
+                                                       (*total_msat, RecipientOnionFields {
+                                                                       payment_secret: *payment_secret,
+                                                                       payment_metadata: payment_metadata.clone(),
+                                                               }, *keysend_preimage)
                                                },
                                                PendingOutboundPayment::Legacy { .. } => {
                                                        log_error!(logger, "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102");
@@ -702,7 +780,7 @@ impl OutboundPayments {
                                        };
                                        if !payment.get().is_retryable_now() {
                                                log_error!(logger, "Retries exhausted for payment id {}", log_bytes!(payment_id.0));
-                                               abandon_with_entry!(payment);
+                                               abandon_with_entry!(payment, PaymentFailureReason::RetriesExhausted);
                                                return
                                        }
                                        payment.get_mut().increment_attempts();
@@ -717,7 +795,7 @@ impl OutboundPayments {
                                }
                        }
                };
-               let res = self.pay_route_internal(&route, payment_hash, &payment_secret, keysend_preimage,
+               let res = self.pay_route_internal(&route, payment_hash, recipient_onion, keysend_preimage,
                        payment_id, Some(total_msat), onion_session_privs, node_signer, best_block_height,
                        &send_payment_along_path);
                log_info!(logger, "Result retrying payment id {}: {:?}", log_bytes!(payment_id.0), res);
@@ -738,16 +816,16 @@ impl OutboundPayments {
                NS::Target: NodeSigner,
                L::Target: Logger,
                IH: Fn() -> InFlightHtlcs,
-               SP: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               SP: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                match err {
                        PaymentSendFailure::AllFailedResendSafe(errs) => {
-                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, errs.into_iter().map(|e| Err(e)), pending_events);
+                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, errs.into_iter().map(|e| Err(e)), logger, pending_events);
                                self.retry_payment_internal(payment_hash, payment_id, route_params, router, first_hops, inflight_htlcs, entropy_source, node_signer, best_block_height, logger, pending_events, send_payment_along_path);
                        },
                        PaymentSendFailure::PartialFailure { failed_paths_retry: Some(mut retry), results, .. } => {
-                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut retry, route.paths, results.into_iter(), pending_events);
+                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut retry, route.paths, results.into_iter(), logger, pending_events);
                                // Some paths were sent, even if we failed to send the full MPP value our recipient may
                                // misbehave and claim the funds, at which point we have to consider the payment sent, so
                                // return `Ok()` here, ignoring any retry errors.
@@ -759,29 +837,31 @@ impl OutboundPayments {
                                // initial HTLC-Add messages yet.
                        },
                        PaymentSendFailure::PathParameterError(results) => {
-                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, results.into_iter(), pending_events);
-                               self.abandon_payment(payment_id, pending_events);
+                               log_error!(logger, "Failed to send to route due to parameter error in a single path. Your router is buggy");
+                               Self::push_path_failed_evs_and_scids(payment_id, payment_hash, &mut route_params, route.paths, results.into_iter(), logger, pending_events);
+                               self.abandon_payment(payment_id, PaymentFailureReason::UnexpectedError, pending_events);
                        },
                        PaymentSendFailure::ParameterError(e) => {
                                log_error!(logger, "Failed to send to route due to parameter error: {:?}. Your router is buggy", e);
-                               self.abandon_payment(payment_id, pending_events);
+                               self.abandon_payment(payment_id, PaymentFailureReason::UnexpectedError, pending_events);
                        },
                        PaymentSendFailure::DuplicatePayment => debug_assert!(false), // unreachable
                }
        }
 
-       fn push_path_failed_evs_and_scids<I: ExactSizeIterator + Iterator<Item = Result<(), APIError>>>(
+       fn push_path_failed_evs_and_scids<I: ExactSizeIterator + Iterator<Item = Result<(), APIError>>, L: Deref>(
                payment_id: PaymentId, payment_hash: PaymentHash, route_params: &mut RouteParameters,
-               paths: Vec<Vec<RouteHop>>, path_results: I, pending_events: &Mutex<Vec<events::Event>>
-       ) {
+               paths: Vec<Path>, path_results: I, logger: &L, pending_events: &Mutex<Vec<events::Event>>
+       ) where L::Target: Logger {
                let mut events = pending_events.lock().unwrap();
                debug_assert_eq!(paths.len(), path_results.len());
                for (path, path_res) in paths.into_iter().zip(path_results) {
                        if let Err(e) = path_res {
                                if let APIError::MonitorUpdateInProgress = e { continue }
+                               log_error!(logger, "Failed to send along path due to error: {:?}", e);
                                let mut failed_scid = None;
                                if let APIError::ChannelUnavailable { .. } = e {
-                                       let scid = path[0].short_channel_id;
+                                       let scid = path.hops[0].short_channel_id;
                                        failed_scid = Some(scid);
                                        route_params.payment_params.previously_failed_channels.push(scid);
                                }
@@ -802,29 +882,33 @@ impl OutboundPayments {
        }
 
        pub(super) fn send_probe<ES: Deref, NS: Deref, F>(
-               &self, hops: Vec<RouteHop>, probing_cookie_secret: [u8; 32], entropy_source: &ES,
-               node_signer: &NS, best_block_height: u32, send_payment_along_path: F
+               &self, path: Path, probing_cookie_secret: [u8; 32], entropy_source: &ES, node_signer: &NS,
+               best_block_height: u32, send_payment_along_path: F
        ) -> Result<(PaymentHash, PaymentId), PaymentSendFailure>
        where
                ES::Target: EntropySource,
                NS::Target: NodeSigner,
-               F: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               F: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                let payment_id = PaymentId(entropy_source.get_secure_random_bytes());
 
                let payment_hash = probing_cookie_from_id(&payment_id, probing_cookie_secret);
 
-               if hops.len() < 2 {
+               if path.hops.len() < 2 && path.blinded_tail.is_none() {
                        return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
                                err: "No need probing a path with less than two hops".to_string()
                        }))
                }
 
-               let route = Route { paths: vec![hops], payment_params: None };
-               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, None, &route, None, None, entropy_source, best_block_height)?;
+               let route = Route { paths: vec![path], payment_params: None };
+               let onion_session_privs = self.add_new_pending_payment(payment_hash,
+                       RecipientOnionFields::spontaneous_empty(), payment_id, None, &route, None, None,
+                       entropy_source, best_block_height)?;
 
-               match self.pay_route_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path) {
+               match self.pay_route_internal(&route, payment_hash, RecipientOnionFields::spontaneous_empty(),
+                       None, payment_id, None, onion_session_privs, node_signer, best_block_height, &send_payment_along_path
+               ) {
                        Ok(()) => Ok((payment_hash, payment_id)),
                        Err(e) => {
                                self.remove_outbound_if_all_failed(payment_id, &e);
@@ -833,16 +917,28 @@ impl OutboundPayments {
                }
        }
 
+       #[cfg(test)]
+       pub(super) fn test_set_payment_metadata(
+               &self, payment_id: PaymentId, new_payment_metadata: Option<Vec<u8>>
+       ) {
+               match self.pending_outbound_payments.lock().unwrap().get_mut(&payment_id).unwrap() {
+                       PendingOutboundPayment::Retryable { payment_metadata, .. } => {
+                               *payment_metadata = new_payment_metadata;
+                       },
+                       _ => panic!("Need a retryable payment to update metadata on"),
+               }
+       }
+
        #[cfg(test)]
        pub(super) fn test_add_new_pending_payment<ES: Deref>(
-               &self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId,
+               &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId,
                route: &Route, retry_strategy: Option<Retry>, entropy_source: &ES, best_block_height: u32
        ) -> Result<Vec<[u8; 32]>, PaymentSendFailure> where ES::Target: EntropySource {
-               self.add_new_pending_payment(payment_hash, payment_secret, payment_id, None, route, retry_strategy, None, entropy_source, best_block_height)
+               self.add_new_pending_payment(payment_hash, recipient_onion, payment_id, None, route, retry_strategy, None, entropy_source, best_block_height)
        }
 
        pub(super) fn add_new_pending_payment<ES: Deref>(
-               &self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId,
+               &self, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields, payment_id: PaymentId,
                keysend_preimage: Option<PaymentPreimage>, route: &Route, retry_strategy: Option<Retry>,
                payment_params: Option<PaymentParameters>, entropy_source: &ES, best_block_height: u32
        ) -> Result<Vec<[u8; 32]>, PaymentSendFailure> where ES::Target: EntropySource {
@@ -863,7 +959,8 @@ impl OutboundPayments {
                                        pending_amt_msat: 0,
                                        pending_fee_msat: Some(0),
                                        payment_hash,
-                                       payment_secret,
+                                       payment_secret: recipient_onion.payment_secret,
+                                       payment_metadata: recipient_onion.payment_metadata,
                                        keysend_preimage,
                                        starting_block_height: best_block_height,
                                        total_msat: route.get_total_amount(),
@@ -879,37 +976,43 @@ impl OutboundPayments {
        }
 
        fn pay_route_internal<NS: Deref, F>(
-               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>,
                onion_session_privs: Vec<[u8; 32]>, node_signer: &NS, best_block_height: u32,
                send_payment_along_path: &F
        ) -> Result<(), PaymentSendFailure>
        where
                NS::Target: NodeSigner,
-               F: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               F: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
                if route.paths.len() < 1 {
                        return Err(PaymentSendFailure::ParameterError(APIError::InvalidRoute{err: "There must be at least one path to send over".to_owned()}));
                }
-               if payment_secret.is_none() && route.paths.len() > 1 {
+               if recipient_onion.payment_secret.is_none() && route.paths.len() > 1 {
                        return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError{err: "Payment secret is required for multi-path payments".to_owned()}));
                }
                let mut total_value = 0;
                let our_node_id = node_signer.get_node_id(Recipient::Node).unwrap(); // TODO no unwrap
                let mut path_errs = Vec::with_capacity(route.paths.len());
                'path_check: for path in route.paths.iter() {
-                       if path.len() < 1 || path.len() > 20 {
+                       if path.hops.len() < 1 || path.hops.len() > 20 {
                                path_errs.push(Err(APIError::InvalidRoute{err: "Path didn't go anywhere/had bogus size".to_owned()}));
                                continue 'path_check;
                        }
-                       for (idx, hop) in path.iter().enumerate() {
-                               if idx != path.len() - 1 && hop.pubkey == our_node_id {
+                       if path.blinded_tail.is_some() {
+                               path_errs.push(Err(APIError::InvalidRoute{err: "Sending to blinded paths isn't supported yet".to_owned()}));
+                               continue 'path_check;
+                       }
+                       let dest_hop_idx = if path.blinded_tail.is_some() && path.blinded_tail.as_ref().unwrap().hops.len() > 1 {
+                               usize::max_value() } else { path.hops.len() - 1 };
+                       for (idx, hop) in path.hops.iter().enumerate() {
+                               if idx != dest_hop_idx && hop.pubkey == our_node_id {
                                        path_errs.push(Err(APIError::InvalidRoute{err: "Path went through us but wasn't a simple rebalance loop to us".to_owned()}));
                                        continue 'path_check;
                                }
                        }
-                       total_value += path.last().unwrap().fee_msat;
+                       total_value += path.final_value_msat();
                        path_errs.push(Ok(()));
                }
                if path_errs.iter().any(|e| e.is_err()) {
@@ -923,7 +1026,8 @@ impl OutboundPayments {
                let mut results = Vec::new();
                debug_assert_eq!(route.paths.len(), onion_session_privs.len());
                for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) {
-                       let mut path_res = send_payment_along_path(&path, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv);
+                       let mut path_res = send_payment_along_path(&path, &payment_hash, recipient_onion.clone(),
+                               total_value, cur_height, payment_id, &keysend_preimage, session_priv);
                        match path_res {
                                Ok(_) => {},
                                Err(APIError::MonitorUpdateInProgress) => {
@@ -947,7 +1051,6 @@ impl OutboundPayments {
                let mut has_ok = false;
                let mut has_err = false;
                let mut pending_amt_unsent = 0;
-               let mut max_unsent_cltv_delta = 0;
                for (res, path) in results.iter().zip(route.paths.iter()) {
                        if res.is_ok() { has_ok = true; }
                        if res.is_err() { has_err = true; }
@@ -957,8 +1060,7 @@ impl OutboundPayments {
                                has_err = true;
                                has_ok = true;
                        } else if res.is_err() {
-                               pending_amt_unsent += path.last().unwrap().fee_msat;
-                               max_unsent_cltv_delta = cmp::max(max_unsent_cltv_delta, path.last().unwrap().cltv_expiry_delta);
+                               pending_amt_unsent += path.final_value_msat();
                        }
                }
                if has_err && has_ok {
@@ -983,17 +1085,17 @@ impl OutboundPayments {
 
        #[cfg(test)]
        pub(super) fn test_send_payment_internal<NS: Deref, F>(
-               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               &self, route: &Route, payment_hash: PaymentHash, recipient_onion: RecipientOnionFields,
                keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>,
                onion_session_privs: Vec<[u8; 32]>, node_signer: &NS, best_block_height: u32,
                send_payment_along_path: F
        ) -> Result<(), PaymentSendFailure>
        where
                NS::Target: NodeSigner,
-               F: Fn(&Vec<RouteHop>, &PaymentHash, &Option<PaymentSecret>, u64, u32, PaymentId,
+               F: Fn(&Path, &PaymentHash, RecipientOnionFields, u64, u32, PaymentId,
                        &Option<PaymentPreimage>, [u8; 32]) -> Result<(), APIError>
        {
-               self.pay_route_internal(route, payment_hash, payment_secret, keysend_preimage, payment_id,
+               self.pay_route_internal(route, payment_hash, recipient_onion, keysend_preimage, payment_id,
                        recv_value_msat, onion_session_privs, node_signer, best_block_height,
                        &send_payment_along_path)
                        .map_err(|e| { self.remove_outbound_if_all_failed(payment_id, &e); e })
@@ -1010,7 +1112,7 @@ impl OutboundPayments {
 
        pub(super) fn claim_htlc<L: Deref>(
                &self, payment_id: PaymentId, payment_preimage: PaymentPreimage, session_priv: SecretKey,
-               path: Vec<RouteHop>, from_onchain: bool, pending_events: &Mutex<Vec<events::Event>>, logger: &L
+               path: Path, from_onchain: bool, pending_events: &Mutex<Vec<events::Event>>, logger: &L
        ) where L::Target: Logger {
                let mut session_priv_bytes = [0; 32];
                session_priv_bytes.copy_from_slice(&session_priv[..]);
@@ -1119,9 +1221,8 @@ impl OutboundPayments {
        // Returns a bool indicating whether a PendingHTLCsForwardable event should be generated.
        pub(super) fn fail_htlc<L: Deref>(
                &self, source: &HTLCSource, payment_hash: &PaymentHash, onion_error: &HTLCFailReason,
-               path: &Vec<RouteHop>, session_priv: &SecretKey, payment_id: &PaymentId,
-               probing_cookie_secret: [u8; 32], secp_ctx: &Secp256k1<secp256k1::All>,
-               pending_events: &Mutex<Vec<events::Event>>, logger: &L
+               path: &Path, session_priv: &SecretKey, payment_id: &PaymentId, probing_cookie_secret: [u8; 32],
+               secp_ctx: &Secp256k1<secp256k1::All>, pending_events: &Mutex<Vec<events::Event>>, logger: &L
        ) -> bool where L::Target: Logger {
                #[cfg(test)]
                let (network_update, short_channel_id, payment_retryable, onion_error_code, onion_error_data) = onion_error.decode_onion_failure(secp_ctx, logger, &source);
@@ -1167,15 +1268,21 @@ impl OutboundPayments {
                        }
 
                        if payment_is_probe || !is_retryable_now || !payment_retryable {
-                               let _ = payment.get_mut().mark_abandoned(); // we'll only Err if it's a legacy payment
+                               let reason = if !payment_retryable {
+                                       PaymentFailureReason::RecipientRejected
+                               } else {
+                                       PaymentFailureReason::RetriesExhausted
+                               };
+                               payment.get_mut().mark_abandoned(reason);
                                is_retryable_now = false;
                        }
                        if payment.get().remaining_parts() == 0 {
-                               if payment.get().abandoned() {
+                               if let PendingOutboundPayment::Abandoned { payment_hash, reason, .. }= payment.get() {
                                        if !payment_is_probe {
                                                full_failure_ev = Some(events::Event::PaymentFailed {
                                                        payment_id: *payment_id,
-                                                       payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
+                                                       payment_hash: *payment_hash,
+                                                       reason: *reason,
                                                });
                                        }
                                        payment.remove();
@@ -1233,15 +1340,17 @@ impl OutboundPayments {
        }
 
        pub(super) fn abandon_payment(
-               &self, payment_id: PaymentId, pending_events: &Mutex<Vec<events::Event>>
+               &self, payment_id: PaymentId, reason: PaymentFailureReason, pending_events: &Mutex<Vec<events::Event>>
        ) {
                let mut outbounds = self.pending_outbound_payments.lock().unwrap();
                if let hash_map::Entry::Occupied(mut payment) = outbounds.entry(payment_id) {
-                       if let Ok(()) = payment.get_mut().mark_abandoned() {
+                       payment.get_mut().mark_abandoned(reason);
+                       if let PendingOutboundPayment::Abandoned { payment_hash, reason, .. } = payment.get() {
                                if payment.get().remaining_parts() == 0 {
                                        pending_events.lock().unwrap().push(events::Event::PaymentFailed {
                                                payment_id,
-                                               payment_hash: payment.get().payment_hash().expect("PendingOutboundPayments::RetriesExceeded always has a payment hash set"),
+                                               payment_hash: *payment_hash,
+                                               reason: *reason,
                                        });
                                        payment.remove();
                                }
@@ -1296,6 +1405,7 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
                (4, payment_secret, option),
                (5, keysend_preimage, option),
                (6, total_msat, required),
+               (7, payment_metadata, option),
                (8, pending_amt_msat, required),
                (10, starting_block_height, required),
                (not_written, retry_strategy, (static_value, None)),
@@ -1303,6 +1413,7 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
        },
        (3, Abandoned) => {
                (0, session_privs, required),
+               (1, reason, option),
                (2, payment_hash, required),
        },
 );
@@ -1312,14 +1423,14 @@ mod tests {
        use bitcoin::network::constants::Network;
        use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
 
-       use crate::events::{Event, PathFailure};
+       use crate::events::{Event, PathFailure, PaymentFailureReason};
        use crate::ln::PaymentHash;
-       use crate::ln::channelmanager::PaymentId;
+       use crate::ln::channelmanager::{PaymentId, RecipientOnionFields};
        use crate::ln::features::{ChannelFeatures, NodeFeatures};
        use crate::ln::msgs::{ErrorAction, LightningError};
        use crate::ln::outbound_payment::{OutboundPayments, Retry, RetryableSendFailure};
        use crate::routing::gossip::NetworkGraph;
-       use crate::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters};
+       use crate::routing::router::{InFlightHtlcs, Path, PaymentParameters, Route, RouteHop, RouteParameters};
        use crate::sync::{Arc, Mutex};
        use crate::util::errors::APIError;
        use crate::util::test_utils;
@@ -1351,20 +1462,24 @@ mod tests {
                };
                let pending_events = Mutex::new(Vec::new());
                if on_retry {
-                       outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None,
-                       &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)),
-                       Some(expired_route_params.payment_params.clone()), &&keys_manager, 0).unwrap();
+                       outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(),
+                               PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None },
+                               Some(Retry::Attempts(1)), Some(expired_route_params.payment_params.clone()),
+                               &&keys_manager, 0).unwrap();
                        outbound_payments.retry_payment_internal(
                                PaymentHash([0; 32]), PaymentId([0; 32]), expired_route_params, &&router, vec![],
                                &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
                                &pending_events, &|_, _, _, _, _, _, _, _| Ok(()));
                        let events = pending_events.lock().unwrap();
                        assert_eq!(events.len(), 1);
-                       if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); }
+                       if let Event::PaymentFailed { ref reason, .. } = events[0] {
+                               assert_eq!(reason.unwrap(), PaymentFailureReason::PaymentExpired);
+                       } else { panic!("Unexpected event"); }
                } else {
                        let err = outbound_payments.send_payment(
-                               PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), expired_route_params,
-                               &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
+                               PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]),
+                               Retry::Attempts(0), expired_route_params, &&router, vec![], || InFlightHtlcs::new(),
+                               &&keys_manager, &&keys_manager, 0, &&logger,
                                &pending_events, |_, _, _, _, _, _, _, _| Ok(())).unwrap_err();
                        if let RetryableSendFailure::PaymentExpired = err { } else { panic!("Unexpected error"); }
                }
@@ -1395,9 +1510,10 @@ mod tests {
 
                let pending_events = Mutex::new(Vec::new());
                if on_retry {
-                       outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), None, PaymentId([0; 32]), None,
-                               &Route { paths: vec![], payment_params: None }, Some(Retry::Attempts(1)),
-                               Some(route_params.payment_params.clone()), &&keys_manager, 0).unwrap();
+                       outbound_payments.add_new_pending_payment(PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(),
+                               PaymentId([0; 32]), None, &Route { paths: vec![], payment_params: None },
+                               Some(Retry::Attempts(1)), Some(route_params.payment_params.clone()),
+                               &&keys_manager, 0).unwrap();
                        outbound_payments.retry_payment_internal(
                                PaymentHash([0; 32]), PaymentId([0; 32]), route_params, &&router, vec![],
                                &|| InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
@@ -1407,8 +1523,9 @@ mod tests {
                        if let Event::PaymentFailed { .. } = events[0] { } else { panic!("Unexpected event"); }
                } else {
                        let err = outbound_payments.send_payment(
-                               PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params,
-                               &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
+                               PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]),
+                               Retry::Attempts(0), route_params, &&router, vec![], || InFlightHtlcs::new(),
+                               &&keys_manager, &&keys_manager, 0, &&logger,
                                &pending_events, |_, _, _, _, _, _, _, _| Ok(())).unwrap_err();
                        if let RetryableSendFailure::RouteNotFound = err {
                        } else { panic!("Unexpected error"); }
@@ -1434,14 +1551,14 @@ mod tests {
                };
                let failed_scid = 42;
                let route = Route {
-                       paths: vec![vec![RouteHop {
+                       paths: vec![Path { hops: vec![RouteHop {
                                pubkey: receiver_pk,
                                node_features: NodeFeatures::empty(),
                                short_channel_id: failed_scid,
                                channel_features: ChannelFeatures::empty(),
                                fee_msat: 0,
                                cltv_expiry_delta: 0,
-                       }]],
+                       }], blinded_tail: None }],
                        payment_params: Some(payment_params),
                };
                router.expect_find_route(route_params.clone(), Ok(route.clone()));
@@ -1455,9 +1572,9 @@ mod tests {
                // PaymentPathFailed event.
                let pending_events = Mutex::new(Vec::new());
                outbound_payments.send_payment(
-                       PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params.clone(),
-                       &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
-                       &pending_events,
+                       PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]),
+                       Retry::Attempts(0), route_params.clone(), &&router, vec![], || InFlightHtlcs::new(),
+                       &&keys_manager, &&keys_manager, 0, &&logger, &pending_events,
                        |_, _, _, _, _, _, _, _| Err(APIError::ChannelUnavailable { err: "test".to_owned() }))
                        .unwrap();
                let mut events = pending_events.lock().unwrap();
@@ -1474,20 +1591,17 @@ mod tests {
 
                // Ensure that a MonitorUpdateInProgress "error" will not result in a PaymentPathFailed event.
                outbound_payments.send_payment(
-                       PaymentHash([0; 32]), &None, PaymentId([0; 32]), Retry::Attempts(0), route_params.clone(),
-                       &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
-                       &pending_events, |_, _, _, _, _, _, _, _| Err(APIError::MonitorUpdateInProgress))
-                       .unwrap();
-               {
-                       let events = pending_events.lock().unwrap();
-                       assert_eq!(events.len(), 0);
-               }
+                       PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]),
+                       Retry::Attempts(0), route_params.clone(), &&router, vec![], || InFlightHtlcs::new(),
+                       &&keys_manager, &&keys_manager, 0, &&logger, &pending_events,
+                       |_, _, _, _, _, _, _, _| Err(APIError::MonitorUpdateInProgress)).unwrap();
+               assert_eq!(pending_events.lock().unwrap().len(), 0);
 
                // Ensure that any other error will result in a PaymentPathFailed event but no blamed scid.
                outbound_payments.send_payment(
-                       PaymentHash([0; 32]), &None, PaymentId([1; 32]), Retry::Attempts(0), route_params.clone(),
-                       &&router, vec![], || InFlightHtlcs::new(), &&keys_manager, &&keys_manager, 0, &&logger,
-                       &pending_events,
+                       PaymentHash([0; 32]), RecipientOnionFields::spontaneous_empty(), PaymentId([1; 32]),
+                       Retry::Attempts(0), route_params.clone(), &&router, vec![], || InFlightHtlcs::new(),
+                       &&keys_manager, &&keys_manager, 0, &&logger, &pending_events,
                        |_, _, _, _, _, _, _, _| Err(APIError::APIMisuseError { err: "test".to_owned() }))
                        .unwrap();
                let events = pending_events.lock().unwrap();
index 1ce0cc0345869dd9a392ebf78944c124d471aefc..69ae4bb81f57d1ab303dd1e04280e72497ead6cd 100644 (file)
 //! payments thereafter.
 
 use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Listen, Watch};
-use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
+use crate::chain::channelmonitor::{ANTI_REORG_DELAY, HTLC_FAIL_BACK_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS};
 use crate::chain::keysinterface::EntropySource;
 use crate::chain::transaction::OutPoint;
-use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure};
+use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider, PathFailure, PaymentFailureReason};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails};
+use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, ChannelManager, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS, RecentPaymentDetails, RecipientOnionFields};
 use crate::ln::features::InvoiceFeatures;
 use crate::ln::msgs;
 use crate::ln::msgs::ChannelMessageHandler;
 use crate::ln::outbound_payment::Retry;
 use crate::routing::gossip::{EffectiveCapacity, RoutingFees};
-use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RouteParameters};
+use crate::routing::router::{get_route, Path, PaymentParameters, Route, Router, RouteHint, RouteHintHop, RouteHop, RouteParameters};
 use crate::routing::scoring::ChannelUsage;
 use crate::util::test_utils;
 use crate::util::errors::APIError;
@@ -59,12 +59,12 @@ fn mpp_failure() {
        let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[3], 100000);
        let path = route.paths[0].clone();
        route.paths.push(path);
-       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-       route.paths[0][0].short_channel_id = chan_1_id;
-       route.paths[0][1].short_channel_id = chan_3_id;
-       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-       route.paths[1][0].short_channel_id = chan_2_id;
-       route.paths[1][1].short_channel_id = chan_4_id;
+       route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0].hops[0].short_channel_id = chan_1_id;
+       route.paths[0].hops[1].short_channel_id = chan_3_id;
+       route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1].hops[0].short_channel_id = chan_2_id;
+       route.paths[1].hops[1].short_channel_id = chan_4_id;
        send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
        fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 }
@@ -87,12 +87,12 @@ fn mpp_retry() {
        let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], amt_msat);
        let path = route.paths[0].clone();
        route.paths.push(path);
-       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-       route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
-       route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
-       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-       route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
-       route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
+       route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
+       route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
+       route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
+       route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 
        // Initiate the MPP payment.
        let payment_id = PaymentId(payment_hash.0);
@@ -102,7 +102,8 @@ fn mpp_retry() {
        };
 
        nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
        check_added_monitors!(nodes[0], 2); // one monitor per path
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 2);
@@ -176,15 +177,16 @@ fn do_mpp_receive_timeout(send_partial_mpp: bool) {
        let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 100_000);
        let path = route.paths[0].clone();
        route.paths.push(path);
-       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
-       route.paths[0][0].short_channel_id = chan_1_update.contents.short_channel_id;
-       route.paths[0][1].short_channel_id = chan_3_update.contents.short_channel_id;
-       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
-       route.paths[1][0].short_channel_id = chan_2_update.contents.short_channel_id;
-       route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
+       route.paths[0].hops[0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0].hops[0].short_channel_id = chan_1_update.contents.short_channel_id;
+       route.paths[0].hops[1].short_channel_id = chan_3_update.contents.short_channel_id;
+       route.paths[1].hops[0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1].hops[0].short_channel_id = chan_2_update.contents.short_channel_id;
+       route.paths[1].hops[1].short_channel_id = chan_4_update.contents.short_channel_id;
 
        // Initiate the MPP payment.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2); // one monitor per path
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 2);
@@ -256,8 +258,9 @@ fn no_pending_leak_on_initial_send_failure() {
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 
-       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
-               true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+               ), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Peer for first hop currently disconnected"));
 
        assert!(!nodes[0].node.has_pending_payments());
@@ -298,7 +301,8 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
                payment_params: route.payment_params.clone().unwrap(),
                final_value_msat: amt_msat,
        };
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -339,7 +343,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        if !confirm_before_reload {
                let as_broadcasted_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                assert_eq!(as_broadcasted_txn.len(), 1);
-               assert_eq!(as_broadcasted_txn[0], as_commitment_tx);
+               assert_eq!(as_broadcasted_txn[0].txid(), as_commitment_tx.txid());
        } else {
                assert!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
        }
@@ -402,9 +406,11 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        mine_transaction(&nodes[0], &bs_htlc_claim_txn[0]);
        expect_payment_sent!(nodes[0], payment_preimage_1);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV*4 + 20);
-       let as_htlc_timeout_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-       assert_eq!(as_htlc_timeout_txn.len(), 2);
-       let (first_htlc_timeout_tx, second_htlc_timeout_tx) = (&as_htlc_timeout_txn[0], &as_htlc_timeout_txn[1]);
+       let (first_htlc_timeout_tx, second_htlc_timeout_tx) = {
+               let mut txn = nodes[0].tx_broadcaster.unique_txn_broadcast();
+               assert_eq!(txn.len(), 2);
+               (txn.remove(0), txn.remove(0))
+       };
        check_spends!(first_htlc_timeout_tx, as_commitment_tx);
        check_spends!(second_htlc_timeout_tx, as_commitment_tx);
        if first_htlc_timeout_tx.input[0].previous_output == bs_htlc_claim_txn[0].input[0].previous_output {
@@ -431,7 +437,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
                let mut new_config = channel.config();
                new_config.forwarding_fee_base_msat += 100_000;
                channel.update_config(&new_config);
-               new_route.paths[0][0].fee_msat += 100_000;
+               new_route.paths[0].hops[0].fee_msat += 100_000;
        }
 
        // Force expiration of the channel's previous config.
@@ -439,14 +445,16 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
                nodes[1].node.timer_tick_occurred();
        }
 
-       assert!(nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
-       nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash, // Shouldn't be allowed to retry a fulfilled payment
+               RecipientOnionFields::secret_only(payment_secret), payment_id_1).is_err());
+       nodes[0].node.send_payment_with_route(&new_route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
        pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000, payment_hash, Some(payment_secret), events.pop().unwrap(), true, None);
        do_claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
-       expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0][0].fee_msat));
+       expect_payment_sent!(nodes[0], payment_preimage, Some(new_route.paths[0].hops[0].fee_msat));
 }
 
 #[test]
@@ -554,8 +562,8 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
        mine_transaction(&nodes[0], &bs_commitment_tx[0]);
        mine_transaction(&nodes[1], &bs_commitment_tx[0]);
        if !use_dust {
-               connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
-               connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1 + (MIN_CLTV_EXPIRY_DELTA as u32));
+               connect_blocks(&nodes[0], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
+               connect_blocks(&nodes[1], TEST_FINAL_CLTV + (MIN_CLTV_EXPIRY_DELTA as u32));
                let as_htlc_timeout = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                check_spends!(as_htlc_timeout[0], bs_commitment_tx[0]);
                assert_eq!(as_htlc_timeout.len(), 1);
@@ -576,7 +584,7 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
        // If we attempt to retry prior to the HTLC-Timeout (or commitment transaction, for dust HTLCs)
        // confirming, we will fail as it's considered still-pending...
        let (new_route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[2], if use_dust { 1_000 } else { 1_000_000 });
-       match nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id) {
+       match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
                Err(PaymentSendFailure::DuplicatePayment) => {},
                _ => panic!("Unexpected error")
        }
@@ -594,7 +602,8 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
        nodes_0_serialized = nodes[0].node.encode();
 
        // After the payment failed, we're free to send it again.
-       assert!(nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id).is_ok());
+       assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
        assert!(!nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 
        reload_node!(nodes[0], test_default_channel_config(), nodes_0_serialized, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], second_persister, second_new_chain_monitor, second_nodes_0_deserialized);
@@ -604,12 +613,13 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 
        // Now resend the payment, delivering the HTLC and actually claiming it this time. This ensures
        // the payment is not (spuriously) listed as still pending.
-       assert!(nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id).is_ok());
+       assert!(nodes[0].node.send_payment_with_route(&new_route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), payment_id).is_ok());
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], if use_dust { 1_000 } else { 1_000_000 }, payment_hash, payment_secret);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage);
 
-       match nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id) {
+       match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
                Err(PaymentSendFailure::DuplicatePayment) => {},
                _ => panic!("Unexpected error")
        }
@@ -626,7 +636,7 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 
        reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 
-       match nodes[0].node.send_payment(&new_route, payment_hash, &Some(payment_secret), payment_id) {
+       match nodes[0].node.send_payment_with_route(&new_route, payment_hash, RecipientOnionFields::secret_only(payment_secret), payment_id) {
                Err(PaymentSendFailure::DuplicatePayment) => {},
                _ => panic!("Unexpected error")
        }
@@ -674,7 +684,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(node_txn.len(), 3);
-       assert_eq!(node_txn[0], node_txn[1]);
+       assert_eq!(node_txn[0].txid(), node_txn[1].txid());
        check_spends!(node_txn[1], funding_tx);
        check_spends!(node_txn[2], node_txn[1]);
        let timeout_txn = vec![node_txn[2].clone()];
@@ -855,7 +865,8 @@ fn get_ldk_payment_preimage() {
                &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
                Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
                amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        // Make sure to use `get_payment_preimage`
@@ -1069,7 +1080,8 @@ fn claimed_send_payment_idempotent() {
                () => {
                        // If we try to resend a new payment with a different payment_hash but with the same
                        // payment_id, it should be rejected.
-                       let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
+                       let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+                               RecipientOnionFields::secret_only(second_payment_secret), payment_id);
                        match send_result {
                                Err(PaymentSendFailure::DuplicatePayment) => {},
                                _ => panic!("Unexpected send result: {:?}", send_result),
@@ -1077,7 +1089,8 @@ fn claimed_send_payment_idempotent() {
 
                        // Further, if we try to send a spontaneous payment with the same payment_id it should
                        // also be rejected.
-                       let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
+                       let send_result = nodes[0].node.send_spontaneous_payment(
+                               &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
                        match send_result {
                                Err(PaymentSendFailure::DuplicatePayment) => {},
                                _ => panic!("Unexpected send result: {:?}", send_result),
@@ -1117,7 +1130,8 @@ fn claimed_send_payment_idempotent() {
                nodes[0].node.timer_tick_occurred();
        }
 
-       nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
+       nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+               RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
        claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
@@ -1141,7 +1155,8 @@ fn abandoned_send_payment_idempotent() {
                () => {
                        // If we try to resend a new payment with a different payment_hash but with the same
                        // payment_id, it should be rejected.
-                       let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
+                       let send_result = nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+                               RecipientOnionFields::secret_only(second_payment_secret), payment_id);
                        match send_result {
                                Err(PaymentSendFailure::DuplicatePayment) => {},
                                _ => panic!("Unexpected send result: {:?}", send_result),
@@ -1149,7 +1164,8 @@ fn abandoned_send_payment_idempotent() {
 
                        // Further, if we try to send a spontaneous payment with the same payment_id it should
                        // also be rejected.
-                       let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
+                       let send_result = nodes[0].node.send_spontaneous_payment(
+                               &route, None, RecipientOnionFields::spontaneous_empty(), payment_id);
                        match send_result {
                                Err(PaymentSendFailure::DuplicatePayment) => {},
                                _ => panic!("Unexpected send result: {:?}", send_result),
@@ -1169,11 +1185,12 @@ fn abandoned_send_payment_idempotent() {
        }
        check_send_rejected!();
 
-       pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
+       pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash, PaymentFailureReason::RecipientRejected);
 
        // However, we can reuse the PaymentId immediately after we `abandon_payment` upon passing the
        // failed payment back.
-       nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
+       nodes[0].node.send_payment_with_route(&route, second_payment_hash,
+               RecipientOnionFields::secret_only(second_payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
        claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
@@ -1322,9 +1339,11 @@ fn test_holding_cell_inflight_htlcs() {
        // Queue up two payments - one will be delivered right away, one immediately goes into the
        // holding cell as nodes[0] is AwaitingRAA.
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_1,
+                       RecipientOnionFields::secret_only(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+               nodes[0].node.send_payment_with_route(&route, payment_hash_2,
+                       RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
        }
 
@@ -1404,7 +1423,8 @@ fn do_test_intercepted_payment(test: InterceptTest) {
        ).unwrap();
 
        let (payment_hash, payment_secret) = nodes[2].node.create_inbound_payment(Some(amt_msat), 60 * 60, None).unwrap();
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        let payment_event = {
                {
                        let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
@@ -1639,7 +1659,8 @@ fn do_automatic_retries(test: AutoRetry) {
 
        if test == AutoRetry::Success {
                // Test that we can succeed on the first retry.
-               nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+               nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                       PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
                pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                // Open a new channel with liquidity on the second hop so we can find a route for the retry
@@ -1654,7 +1675,9 @@ fn do_automatic_retries(test: AutoRetry) {
                pass_along_path(&nodes[0], &[&nodes[1], &nodes[2]], amt_msat, payment_hash, Some(payment_secret), msg_events.pop().unwrap(), true, None);
                claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
        } else if test == AutoRetry::Spontaneous {
-               nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+               nodes[0].node.send_spontaneous_payment_with_retry(Some(payment_preimage),
+                       RecipientOnionFields::spontaneous_empty(), PaymentId(payment_hash.0), route_params,
+                       Retry::Attempts(1)).unwrap();
                pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                // Open a new channel with liquidity on the second hop so we can find a route for the retry
@@ -1670,7 +1693,8 @@ fn do_automatic_retries(test: AutoRetry) {
                claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], false, payment_preimage);
        } else if test == AutoRetry::FailAttempts {
                // Ensure ChannelManager will not retry a payment if it has run out of payment attempts.
-               nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+               nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                       PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
                pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
@@ -1688,7 +1712,8 @@ fn do_automatic_retries(test: AutoRetry) {
        } else if test == AutoRetry::FailTimeout {
                #[cfg(not(feature = "no-std"))] {
                        // Ensure ChannelManager will not retry a payment if it times out due to Retry::Timeout.
-                       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
+                       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                               PaymentId(payment_hash.0), route_params, Retry::Timeout(Duration::from_secs(60))).unwrap();
                        pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                        // Advance the time so the second attempt fails due to timeout.
@@ -1702,9 +1727,10 @@ fn do_automatic_retries(test: AutoRetry) {
                        let mut events = nodes[0].node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
                        match events[0] {
-                               Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+                               Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
                                        assert_eq!(payment_hash, *ev_payment_hash);
                                        assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+                                       assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
                                },
                                _ => panic!("Unexpected event"),
                        }
@@ -1712,7 +1738,8 @@ fn do_automatic_retries(test: AutoRetry) {
        } else if test == AutoRetry::FailOnRestart {
                // Ensure ChannelManager will not retry a payment after restart, even if there were retry
                // attempts remaining prior to restart.
-               nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
+               nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                       PaymentId(payment_hash.0), route_params, Retry::Attempts(2)).unwrap();
                pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                // Open a new channel with no liquidity on the second hop so we can find a (bad) route for
@@ -1737,14 +1764,16 @@ fn do_automatic_retries(test: AutoRetry) {
                let mut events = nodes[0].node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+                       Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
                                assert_eq!(payment_hash, *ev_payment_hash);
                                assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+                               assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
                        },
                        _ => panic!("Unexpected event"),
                }
        } else if test == AutoRetry::FailOnRetry {
-               nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+               nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+                       PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
                pass_failed_attempt_with_retry_along_path!(channel_id_2, true);
 
                // We retry payments in `process_pending_htlc_forwards`. Since our channel closed, we should
@@ -1756,9 +1785,10 @@ fn do_automatic_retries(test: AutoRetry) {
                let mut events = nodes[0].node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+                       Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
                                assert_eq!(payment_hash, *ev_payment_hash);
                                assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+                               assert_eq!(PaymentFailureReason::RouteNotFound, ev_reason.unwrap());
                        },
                        _ => panic!("Unexpected event"),
                }
@@ -1809,56 +1839,56 @@ fn auto_retry_partial_failure() {
        // Configure the initial send, retry1 and retry2's paths.
        let send_route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_id,
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: amt_msat / 2,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_2_id,
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: amt_msat / 2,
                                cltv_expiry_delta: 100,
-                       }],
+                       }], blinded_tail: None },
                ],
                payment_params: Some(route_params.payment_params.clone()),
        };
        let retry_1_route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_id,
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: amt_msat / 4,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_3_id,
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: amt_msat / 4,
                                cltv_expiry_delta: 100,
-                       }],
+                       }], blinded_tail: None },
                ],
                payment_params: Some(route_params.payment_params.clone()),
        };
        let retry_2_route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_id,
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: amt_msat / 4,
                                cltv_expiry_delta: 100,
-                       }],
+                       }], blinded_tail: None },
                ],
                payment_params: Some(route_params.payment_params.clone()),
        };
@@ -1875,7 +1905,8 @@ fn auto_retry_partial_failure() {
                }, Ok(retry_2_route));
 
        // Send a payment that will partially fail on send, then partially fail on retry, then succeed.
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(3)).unwrap();
        let closed_chan_events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(closed_chan_events.len(), 4);
        match closed_chan_events[0] {
@@ -2008,7 +2039,8 @@ fn auto_retry_zero_attempts_send_error() {
        };
 
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(0)).unwrap();
        assert_eq!(nodes[0].node.get_and_clear_pending_msg_events().len(), 2); // channel close messages
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 3);
@@ -2046,7 +2078,8 @@ fn fails_paying_after_rejected_by_payee() {
                final_value_msat: amt_msat,
        };
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -2059,7 +2092,7 @@ fn fails_paying_after_rejected_by_payee() {
 
        nodes[1].node.fail_htlc_backwards(&payment_hash);
        expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash }]);
-       pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash);
+       pass_failed_payment_back(&nodes[0], &[&[&nodes[1]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
 }
 
 #[test]
@@ -2095,29 +2128,29 @@ fn retry_multi_path_single_failed_payment() {
        let chans = nodes[0].node.list_usable_channels();
        let mut route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chans[0].short_channel_id.unwrap(),
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: 10_000,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chans[1].short_channel_id.unwrap(),
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
                                cltv_expiry_delta: 100,
-                       }],
+                       }], blinded_tail: None },
                ],
                payment_params: Some(payment_params),
        };
        nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
        // On retry, split the payment across both channels.
-       route.paths[0][0].fee_msat = 50_000_001;
-       route.paths[1][0].fee_msat = 50_000_000;
+       route.paths[0].hops[0].fee_msat = 50_000_001;
+       route.paths[1].hops[0].fee_msat = 50_000_000;
        let mut pay_params = route.payment_params.clone().unwrap();
        pay_params.previously_failed_channels.push(chans[1].short_channel_id.unwrap());
        nodes[0].router.expect_find_route(RouteParameters {
@@ -2137,7 +2170,8 @@ fn retry_multi_path_single_failed_payment() {
                scorer.expect_usage(chans[1].short_channel_id.unwrap(), ChannelUsage { amount_msat: 50_000_000, inflight_htlc_msat: 0, effective_capacity: EffectiveCapacity::Unknown });
        }
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
@@ -2146,7 +2180,7 @@ fn retry_multi_path_single_failed_payment() {
                        short_channel_id: Some(expected_scid), .. } =>
                {
                        assert_eq!(payment_hash, ev_payment_hash);
-                       assert_eq!(expected_scid, route.paths[1][0].short_channel_id);
+                       assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
                        assert!(err_msg.contains("max HTLC"));
                },
                _ => panic!("Unexpected event"),
@@ -2188,30 +2222,31 @@ fn immediate_retry_on_failure() {
        let chans = nodes[0].node.list_usable_channels();
        let mut route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chans[0].short_channel_id.unwrap(),
                                channel_features: nodes[1].node.channel_features(),
                                fee_msat: 100_000_001, // Our default max-HTLC-value is 10% of the channel value, which this is one more than
                                cltv_expiry_delta: 100,
-                       }],
+                       }], blinded_tail: None },
                ],
                payment_params: Some(PaymentParameters::from_node_id(nodes[1].node.get_our_node_id(), TEST_FINAL_CLTV)),
        };
        nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
        // On retry, split the payment across both channels.
        route.paths.push(route.paths[0].clone());
-       route.paths[0][0].short_channel_id = chans[1].short_channel_id.unwrap();
-       route.paths[0][0].fee_msat = 50_000_000;
-       route.paths[1][0].fee_msat = 50_000_001;
+       route.paths[0].hops[0].short_channel_id = chans[1].short_channel_id.unwrap();
+       route.paths[0].hops[0].fee_msat = 50_000_000;
+       route.paths[1].hops[0].fee_msat = 50_000_001;
        let mut pay_params = route_params.payment_params.clone();
        pay_params.previously_failed_channels.push(chans[0].short_channel_id.unwrap());
        nodes[0].router.expect_find_route(RouteParameters {
                        payment_params: pay_params, final_value_msat: amt_msat,
                }, Ok(route.clone()));
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match events[0] {
@@ -2220,7 +2255,7 @@ fn immediate_retry_on_failure() {
                        short_channel_id: Some(expected_scid), .. } =>
                {
                        assert_eq!(payment_hash, ev_payment_hash);
-                       assert_eq!(expected_scid, route.paths[1][0].short_channel_id);
+                       assert_eq!(expected_scid, route.paths[1].hops[0].short_channel_id);
                        assert!(err_msg.contains("max HTLC"));
                },
                _ => panic!("Unexpected event"),
@@ -2275,7 +2310,7 @@ fn no_extra_retries_on_back_to_back_fail() {
 
        let mut route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_scid,
@@ -2289,8 +2324,8 @@ fn no_extra_retries_on_back_to_back_fail() {
                                channel_features: nodes[2].node.channel_features(),
                                fee_msat: 100_000_000,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_scid,
@@ -2304,7 +2339,7 @@ fn no_extra_retries_on_back_to_back_fail() {
                                channel_features: nodes[2].node.channel_features(),
                                fee_msat: 100_000_000,
                                cltv_expiry_delta: 100,
-                       }]
+                       }], blinded_tail: None }
                ],
                payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
        };
@@ -2313,13 +2348,14 @@ fn no_extra_retries_on_back_to_back_fail() {
        second_payment_params.previously_failed_channels = vec![chan_2_scid, chan_2_scid];
        // On retry, we'll only return one path
        route.paths.remove(1);
-       route.paths[0][1].fee_msat = amt_msat;
+       route.paths[0].hops[1].fee_msat = amt_msat;
        nodes[0].router.expect_find_route(RouteParameters {
                        payment_params: second_payment_params,
                        final_value_msat: amt_msat,
                }, Ok(route.clone()));
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        let htlc_updates = SendEvent::from_node(&nodes[0]);
        check_added_monitors!(nodes[0], 1);
        assert_eq!(htlc_updates.msgs.len(), 1);
@@ -2432,9 +2468,10 @@ fn no_extra_retries_on_back_to_back_fail() {
                _ => panic!("Unexpected event"),
        }
        match events[1] {
-               Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id } => {
+               Event::PaymentFailed { payment_hash: ref ev_payment_hash, payment_id: ref ev_payment_id, reason: ref ev_reason } => {
                        assert_eq!(payment_hash, *ev_payment_hash);
                        assert_eq!(PaymentId(payment_hash.0), *ev_payment_id);
+                       assert_eq!(PaymentFailureReason::RetriesExhausted, ev_reason.unwrap());
                },
                _ => panic!("Unexpected event"),
        }
@@ -2475,7 +2512,7 @@ fn test_simple_partial_retry() {
 
        let mut route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_scid,
@@ -2489,8 +2526,8 @@ fn test_simple_partial_retry() {
                                channel_features: nodes[2].node.channel_features(),
                                fee_msat: 100_000_000,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_scid,
@@ -2504,7 +2541,7 @@ fn test_simple_partial_retry() {
                                channel_features: nodes[2].node.channel_features(),
                                fee_msat: 100_000_000,
                                cltv_expiry_delta: 100,
-                       }]
+                       }], blinded_tail: None }
                ],
                payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
        };
@@ -2518,7 +2555,8 @@ fn test_simple_partial_retry() {
                        final_value_msat: amt_msat / 2,
                }, Ok(route.clone()));
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params, Retry::Attempts(1)).unwrap();
        let htlc_updates = SendEvent::from_node(&nodes[0]);
        check_added_monitors!(nodes[0], 1);
        assert_eq!(htlc_updates.msgs.len(), 1);
@@ -2640,7 +2678,7 @@ fn test_threaded_payment_retries() {
 
        let mut route = Route {
                paths: vec![
-                       vec![RouteHop {
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[1].node.get_our_node_id(),
                                node_features: nodes[1].node.node_features(),
                                short_channel_id: chan_1_scid,
@@ -2654,8 +2692,8 @@ fn test_threaded_payment_retries() {
                                channel_features: nodes[2].node.channel_features(),
                                fee_msat: amt_msat / 1000,
                                cltv_expiry_delta: 100,
-                       }],
-                       vec![RouteHop {
+                       }], blinded_tail: None },
+                       Path { hops: vec![RouteHop {
                                pubkey: nodes[2].node.get_our_node_id(),
                                node_features: nodes[2].node.node_features(),
                                short_channel_id: chan_3_scid,
@@ -2669,13 +2707,14 @@ fn test_threaded_payment_retries() {
                                channel_features: nodes[3].node.channel_features(),
                                fee_msat: amt_msat - amt_msat / 1000,
                                cltv_expiry_delta: 100,
-                       }]
+                       }], blinded_tail: None }
                ],
                payment_params: Some(PaymentParameters::from_node_id(nodes[2].node.get_our_node_id(), TEST_FINAL_CLTV)),
        };
        nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
 
-       nodes[0].node.send_payment_with_retry(payment_hash, &Some(payment_secret), PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(0xdeadbeef)).unwrap();
        check_added_monitors!(nodes[0], 2);
        let mut send_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(send_msg_events.len(), 2);
@@ -2726,9 +2765,9 @@ fn test_threaded_payment_retries() {
                // we should still ultimately fail for the same reason - because we're trying to send too
                // many HTLCs at once.
                let mut new_route_params = route_params.clone();
-               previously_failed_channels.push(route.paths[0][1].short_channel_id);
+               previously_failed_channels.push(route.paths[0].hops[1].short_channel_id);
                new_route_params.payment_params.previously_failed_channels = previously_failed_channels.clone();
-               route.paths[0][1].short_channel_id += 1;
+               route.paths[0].hops[1].short_channel_id += 1;
                nodes[0].router.expect_find_route(new_route_params, Ok(route.clone()));
 
                let bs_fail_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
@@ -2838,3 +2877,293 @@ fn no_missing_sent_on_midpoint_reload() {
        do_no_missing_sent_on_midpoint_reload(false);
        do_no_missing_sent_on_midpoint_reload(true);
 }
+
+fn do_claim_from_closed_chan(fail_payment: bool) {
+       // Previously, LDK would refuse to claim a payment if a channel on which the payment was
+       // received had been closed between when the HTLC was received and when we went to claim it.
+       // This makes sense in the payment case - why pay an on-chain fee to claim the HTLC when
+       // presumably the sender may retry later. Long ago it also reduced total code in the claim
+       // pipeline.
+       //
+       // However, this doesn't make sense if you're trying to do an atomic swap or some other
+       // protocol that requires atomicity with some other action - if your money got claimed
+       // elsewhere you need to be able to claim the HTLC in lightning no matter what. Further, this
+       // is an over-optimization - there should be a very, very low likelihood that a channel closes
+       // between when we receive the last HTLC for a payment and the user goes to claim the payment.
+       // Since we now have code to handle this anyway we should allow it.
+
+       // Build 4 nodes and send an MPP payment across two paths. By building a route manually set the
+       // CLTVs on the paths to different value resulting in a different claim deadline.
+       let chanmon_cfgs = create_chanmon_cfgs(4);
+       let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
+       let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+
+       create_announced_chan_between_nodes(&nodes, 0, 1);
+       create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
+       let chan_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
+       create_announced_chan_between_nodes(&nodes, 2, 3);
+
+       let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3]);
+       let mut route_params = RouteParameters {
+               payment_params: PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
+                       .with_features(nodes[1].node.invoice_features()),
+               final_value_msat: 10_000_000,
+       };
+       let mut route = nodes[0].router.find_route(&nodes[0].node.get_our_node_id(), &route_params,
+               None, &nodes[0].node.compute_inflight_htlcs()).unwrap();
+       // Make sure the route is ordered as the B->D path before C->D
+       route.paths.sort_by(|a, _| if a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
+               std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater });
+
+       // Note that we add an extra 1 in the send pipeline to compensate for any blocks found while
+       // the HTLC is being relayed.
+       route.paths[0].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 8;
+       route.paths[1].hops[1].cltv_expiry_delta = TEST_FINAL_CLTV + 12;
+       let final_cltv = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 8 + 1;
+
+       nodes[0].router.expect_find_route(route_params.clone(), Ok(route.clone()));
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields::secret_only(payment_secret),
+               PaymentId(payment_hash.0), route_params.clone(), Retry::Attempts(1)).unwrap();
+       check_added_monitors(&nodes[0], 2);
+       let mut send_msgs = nodes[0].node.get_and_clear_pending_msg_events();
+       send_msgs.sort_by(|a, _| {
+               let a_node_id =
+                       if let MessageSendEvent::UpdateHTLCs { node_id, .. } = a { node_id } else { panic!() };
+               let node_b_id = nodes[1].node.get_our_node_id();
+               if *a_node_id == node_b_id { std::cmp::Ordering::Less } else { std::cmp::Ordering::Greater }
+       });
+
+       assert_eq!(send_msgs.len(), 2);
+       pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 10_000_000,
+               payment_hash, Some(payment_secret), send_msgs.remove(0), false, None);
+       let receive_event = pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 10_000_000,
+               payment_hash, Some(payment_secret), send_msgs.remove(0), true, None);
+
+       match receive_event.unwrap() {
+               Event::PaymentClaimable { claim_deadline, .. } => {
+                       assert_eq!(claim_deadline.unwrap(), final_cltv - HTLC_FAIL_BACK_BUFFER);
+               },
+               _ => panic!(),
+       }
+
+       // Ensure that the claim_deadline is correct, with the payment failing at exactly the given
+       // height.
+       connect_blocks(&nodes[3], final_cltv - HTLC_FAIL_BACK_BUFFER - nodes[3].best_block_info().1
+               - if fail_payment { 0 } else { 2 });
+       if fail_payment {
+               // We fail the HTLC on the A->B->D path first as it expires 4 blocks earlier. We go ahead
+               // and expire both immediately, though, by connecting another 4 blocks.
+               let reason = HTLCDestination::FailedPayment { payment_hash };
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason.clone()]);
+               connect_blocks(&nodes[3], 4);
+               expect_pending_htlcs_forwardable_and_htlc_handling_failed!(&nodes[3], [reason]);
+               pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash, PaymentFailureReason::RecipientRejected);
+       } else {
+               nodes[1].node.force_close_broadcasting_latest_txn(&chan_bd, &nodes[3].node.get_our_node_id()).unwrap();
+               check_closed_event(&nodes[1], 1, ClosureReason::HolderForceClosed, false);
+               check_closed_broadcast(&nodes[1], 1, true);
+               let bs_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+               assert_eq!(bs_tx.len(), 1);
+
+               mine_transaction(&nodes[3], &bs_tx[0]);
+               check_added_monitors(&nodes[3], 1);
+               check_closed_broadcast(&nodes[3], 1, true);
+               check_closed_event(&nodes[3], 1, ClosureReason::CommitmentTxConfirmed, false);
+
+               nodes[3].node.claim_funds(payment_preimage);
+               check_added_monitors(&nodes[3], 2);
+               expect_payment_claimed!(nodes[3], payment_hash, 10_000_000);
+
+               let ds_tx = nodes[3].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+               assert_eq!(ds_tx.len(), 1);
+               check_spends!(&ds_tx[0], &bs_tx[0]);
+
+               mine_transactions(&nodes[1], &[&bs_tx[0], &ds_tx[0]]);
+               check_added_monitors(&nodes[1], 1);
+               expect_payment_forwarded!(nodes[1], nodes[0], nodes[3], Some(1000), false, true);
+
+               let bs_claims = nodes[1].node.get_and_clear_pending_msg_events();
+               check_added_monitors(&nodes[1], 1);
+               assert_eq!(bs_claims.len(), 1);
+               if let MessageSendEvent::UpdateHTLCs { updates, .. } = &bs_claims[0] {
+                       nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
+                       commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
+               } else { panic!(); }
+
+               expect_payment_sent!(nodes[0], payment_preimage);
+
+               let ds_claim_msgs = nodes[3].node.get_and_clear_pending_msg_events();
+               assert_eq!(ds_claim_msgs.len(), 1);
+               let cs_claim_msgs = if let MessageSendEvent::UpdateHTLCs { updates, .. } = &ds_claim_msgs[0] {
+                       nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
+                       let cs_claim_msgs = nodes[2].node.get_and_clear_pending_msg_events();
+                       check_added_monitors(&nodes[2], 1);
+                       commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
+                       expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
+                       cs_claim_msgs
+               } else { panic!(); };
+
+               assert_eq!(cs_claim_msgs.len(), 1);
+               if let MessageSendEvent::UpdateHTLCs { updates, .. } = &cs_claim_msgs[0] {
+                       nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
+                       commitment_signed_dance!(nodes[0], nodes[2], updates.commitment_signed, false, true);
+               } else { panic!(); }
+
+               expect_payment_path_successful!(nodes[0]);
+       }
+}
+
+#[test]
+fn claim_from_closed_chan() {
+       do_claim_from_closed_chan(true);
+       do_claim_from_closed_chan(false);
+}
+
+fn do_test_payment_metadata_consistency(do_reload: bool, do_modify: bool) {
+       // Check that a payment metadata received on one HTLC that doesn't match the one received on
+       // another results in the HTLC being rejected.
+       //
+       // We first set up a diamond shaped network, allowing us to split a payment into two HTLCs, the
+       // first of which we'll deliver and the second of which we'll fail and then re-send with
+       // modified payment metadata, which will in turn result in it being failed by the recipient.
+       let chanmon_cfgs = create_chanmon_cfgs(4);
+       let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
+       let mut config = test_default_channel_config();
+       config.channel_handshake_config.max_inbound_htlc_value_in_flight_percent_of_channel = 50;
+       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, Some(config), Some(config), Some(config)]);
+
+       let persister;
+       let new_chain_monitor;
+       let nodes_0_deserialized;
+
+       let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+
+       create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 0);
+       let chan_id_bd = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 1_000_000, 0).2;
+       create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 1_000_000, 0);
+       let chan_id_cd = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
+
+       // Pay more than half of each channel's max, requiring MPP
+       let amt_msat = 750_000_000;
+       let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[3], Some(amt_msat));
+       let payment_id = PaymentId(payment_hash.0);
+       let payment_metadata = vec![44, 49, 52, 142];
+
+       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id(), TEST_FINAL_CLTV)
+               .with_features(nodes[1].node.invoice_features());
+       let mut route_params = RouteParameters {
+               payment_params,
+               final_value_msat: amt_msat,
+       };
+
+       // Send the MPP payment, delivering the updated commitment state to nodes[1].
+       nodes[0].node.send_payment(payment_hash, RecipientOnionFields {
+                       payment_secret: Some(payment_secret), payment_metadata: Some(payment_metadata),
+               }, payment_id, route_params.clone(), Retry::Attempts(1)).unwrap();
+       check_added_monitors!(nodes[0], 2);
+
+       let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
+       assert_eq!(send_events.len(), 2);
+       let first_send = SendEvent::from_event(send_events.pop().unwrap());
+       let second_send = SendEvent::from_event(send_events.pop().unwrap());
+
+       let (b_recv_ev, c_recv_ev) = if first_send.node_id == nodes[1].node.get_our_node_id() {
+               (&first_send, &second_send)
+       } else {
+               (&second_send, &first_send)
+       };
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &b_recv_ev.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], b_recv_ev.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[1]);
+       check_added_monitors(&nodes[1], 1);
+       let b_forward_ev = SendEvent::from_node(&nodes[1]);
+       nodes[3].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &b_forward_ev.msgs[0]);
+       commitment_signed_dance!(nodes[3], nodes[1], b_forward_ev.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[3]);
+
+       // Before delivering the second MPP HTLC to nodes[2], disconnect nodes[2] and nodes[3], which
+       // will result in nodes[2] failing the HTLC back.
+       nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
+       nodes[3].node.peer_disconnected(&nodes[2].node.get_our_node_id());
+
+       nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &c_recv_ev.msgs[0]);
+       commitment_signed_dance!(nodes[2], nodes[0], c_recv_ev.commitment_msg, false, true);
+
+       let cs_fail = get_htlc_update_msgs(&nodes[2], &nodes[0].node.get_our_node_id());
+       nodes[0].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail.update_fail_htlcs[0]);
+       commitment_signed_dance!(nodes[0], nodes[2], cs_fail.commitment_signed, false, true);
+
+       let payment_fail_retryable_evs = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(payment_fail_retryable_evs.len(), 2);
+       if let Event::PaymentPathFailed { .. } = payment_fail_retryable_evs[0] {} else { panic!(); }
+       if let Event::PendingHTLCsForwardable { .. } = payment_fail_retryable_evs[1] {} else { panic!(); }
+
+       // Before we allow the HTLC to be retried, optionally change the payment_metadata we have
+       // stored for our payment.
+       if do_modify {
+               nodes[0].node.test_set_payment_metadata(payment_id, Some(Vec::new()));
+       }
+
+       // Optionally reload nodes[3] to check that the payment_metadata is properly serialized with
+       // the payment state.
+       if do_reload {
+               let mon_bd = get_monitor!(nodes[3], chan_id_bd).encode();
+               let mon_cd = get_monitor!(nodes[3], chan_id_cd).encode();
+               reload_node!(nodes[3], config, &nodes[3].node.encode(), &[&mon_bd, &mon_cd],
+                       persister, new_chain_monitor, nodes_0_deserialized);
+               nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
+               reconnect_nodes(&nodes[1], &nodes[3], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+       }
+       reconnect_nodes(&nodes[2], &nodes[3], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       // Create a new channel between C and D as A will refuse to retry on the existing one because
+       // it just failed.
+       let chan_id_cd_2 = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 1_000_000, 0).2;
+
+       // Now retry the failed HTLC.
+       nodes[0].node.process_pending_htlc_forwards();
+       check_added_monitors(&nodes[0], 1);
+       let as_resend = SendEvent::from_node(&nodes[0]);
+       nodes[2].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_resend.msgs[0]);
+       commitment_signed_dance!(nodes[2], nodes[0], as_resend.commitment_msg, false, true);
+
+       expect_pending_htlcs_forwardable!(nodes[2]);
+       check_added_monitors(&nodes[2], 1);
+       let cs_forward = SendEvent::from_node(&nodes[2]);
+       nodes[3].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &cs_forward.msgs[0]);
+       commitment_signed_dance!(nodes[3], nodes[2], cs_forward.commitment_msg, false, true);
+
+       // Finally, check that nodes[3] does the correct thing - either accepting the payment or, if
+       // the payment metadata was modified, failing only the one modified HTLC and retaining the
+       // other.
+       if do_modify {
+               expect_pending_htlcs_forwardable_ignore!(nodes[3]);
+               nodes[3].node.process_pending_htlc_forwards();
+               expect_pending_htlcs_forwardable_conditions(nodes[3].node.get_and_clear_pending_events(),
+                       &[HTLCDestination::FailedPayment {payment_hash}]);
+               nodes[3].node.process_pending_htlc_forwards();
+
+               check_added_monitors(&nodes[3], 1);
+               let ds_fail = get_htlc_update_msgs(&nodes[3], &nodes[2].node.get_our_node_id());
+
+               nodes[2].node.handle_update_fail_htlc(&nodes[3].node.get_our_node_id(), &ds_fail.update_fail_htlcs[0]);
+               commitment_signed_dance!(nodes[2], nodes[3], ds_fail.commitment_signed, false, true);
+               expect_pending_htlcs_forwardable_conditions(nodes[2].node.get_and_clear_pending_events(),
+                       &[HTLCDestination::NextHopChannel { node_id: Some(nodes[3].node.get_our_node_id()), channel_id: chan_id_cd_2 }]);
+       } else {
+               expect_pending_htlcs_forwardable!(nodes[3]);
+               expect_payment_claimable!(nodes[3], payment_hash, payment_secret, amt_msat);
+               claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
+       }
+}
+
+#[test]
+fn test_payment_metadata_consistency() {
+       do_test_payment_metadata_consistency(true, true);
+       do_test_payment_metadata_consistency(true, false);
+       do_test_payment_metadata_consistency(false, true);
+       do_test_payment_metadata_consistency(false, false);
+}
index bb4b076b1bef313c72486a5d00b3494b66fdbaef..84870143008a3f19bd47fe2a303a897760bba516 100644 (file)
@@ -28,7 +28,7 @@ use crate::ln::peer_channel_encryptor::{PeerChannelEncryptor,NextNoiseStep};
 use crate::ln::wire;
 use crate::ln::wire::Encode;
 use crate::onion_message::{CustomOnionMessageContents, CustomOnionMessageHandler, SimpleArcOnionMessenger, SimpleRefOnionMessenger};
-use crate::routing::gossip::{NetworkGraph, P2PGossipSync, NodeId};
+use crate::routing::gossip::{NetworkGraph, P2PGossipSync, NodeId, NodeAlias};
 use crate::util::atomic_counter::AtomicCounter;
 use crate::util::logger::Logger;
 
@@ -1056,7 +1056,7 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, OM: Deref, L: Deref, CM
                match self.do_read_event(peer_descriptor, data) {
                        Ok(res) => Ok(res),
                        Err(e) => {
-                               log_trace!(self.logger, "Peer sent invalid data or we decided to disconnect due to a protocol error");
+                               log_trace!(self.logger, "Disconnecting peer due to a protocol error (usually a duplicate connection).");
                                self.disconnect_event_internal(peer_descriptor);
                                Err(e)
                        }
@@ -2103,7 +2103,7 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, OM: Deref, L: Deref, CM
                                                if let Some((node_id, _)) = peer.their_node_id {
                                                        self.node_id_to_descriptor.lock().unwrap().remove(&node_id);
                                                }
-                                               self.do_disconnect(descriptor, &*peer, "ping timeout");
+                                               self.do_disconnect(descriptor, &*peer, "ping/handshake timeout");
                                        }
                                }
                        }
@@ -2153,7 +2153,9 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, OM: Deref, L: Deref, CM
                        features,
                        timestamp: self.last_node_announcement_serial.fetch_add(1, Ordering::AcqRel),
                        node_id: NodeId::from_pubkey(&self.node_signer.get_node_id(Recipient::Node).unwrap()),
-                       rgb, alias, addresses,
+                       rgb,
+                       alias: NodeAlias(alias),
+                       addresses,
                        excess_address_data: Vec::new(),
                        excess_data: Vec::new(),
                };
index 6d1df42d18e20e3bb0c7eaf1223b8343ea06a1d3..7bb1fd44f2fe1f6d42698a222d47039f431d91be 100644 (file)
@@ -14,7 +14,7 @@
 use crate::chain::ChannelMonitorUpdateStatus;
 use crate::chain::keysinterface::NodeSigner;
 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
-use crate::ln::channelmanager::{ChannelManager, MIN_CLTV_EXPIRY_DELTA, PaymentId};
+use crate::ln::channelmanager::{ChannelManager, MIN_CLTV_EXPIRY_DELTA, PaymentId, RecipientOnionFields};
 use crate::routing::gossip::RoutingFees;
 use crate::routing::router::{PaymentParameters, RouteHint, RouteHintHop};
 use crate::ln::features::ChannelTypeFeatures;
@@ -71,7 +71,8 @@ fn test_priv_forwarding_rejection() {
                .with_route_hints(last_hops);
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 10_000, TEST_FINAL_CLTV);
 
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let payment_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
@@ -118,7 +119,8 @@ fn test_priv_forwarding_rejection() {
        get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[2].node.get_our_node_id());
        get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[1].node.get_our_node_id());
 
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 10_000, our_payment_hash, our_payment_secret);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], our_payment_preimage);
@@ -237,8 +239,9 @@ fn test_routed_scid_alias() {
                .with_features(nodes[2].node.invoice_features())
                .with_route_hints(hop_hints);
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 100_000, 42);
-       assert_eq!(route.paths[0][1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       assert_eq!(route.paths[0].hops[1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 100_000, payment_hash, payment_secret);
@@ -362,7 +365,10 @@ fn test_inbound_scid_privacy() {
        check_added_monitors!(nodes[2], 1);
 
        let cs_funding_signed = get_event_msg!(nodes[2], MessageSendEvent::SendFundingSigned, nodes[1].node.get_our_node_id());
+       expect_channel_pending_event(&nodes[2], &nodes[1].node.get_our_node_id());
+
        nodes[1].node.handle_funding_signed(&nodes[2].node.get_our_node_id(), &cs_funding_signed);
+       expect_channel_pending_event(&nodes[1], &nodes[2].node.get_our_node_id());
        check_added_monitors!(nodes[1], 1);
 
        let conf_height = core::cmp::max(nodes[1].best_block_info().1 + 1, nodes[2].best_block_info().1 + 1);
@@ -399,8 +405,9 @@ fn test_inbound_scid_privacy() {
                .with_features(nodes[2].node.invoice_features())
                .with_route_hints(hop_hints.clone());
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 100_000, 42);
-       assert_eq!(route.paths[0][1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       assert_eq!(route.paths[0].hops[1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 100_000, payment_hash, payment_secret);
@@ -414,8 +421,9 @@ fn test_inbound_scid_privacy() {
                .with_features(nodes[2].node.invoice_features())
                .with_route_hints(hop_hints);
        let (route_2, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params_2, 100_000, 42);
-       assert_eq!(route_2.paths[0][1].short_channel_id, last_hop[0].short_channel_id.unwrap());
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       assert_eq!(route_2.paths[0].hops[1].short_channel_id, last_hop[0].short_channel_id.unwrap());
+       nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let payment_event = SendEvent::from_node(&nodes[0]);
@@ -465,12 +473,13 @@ fn test_scid_alias_returned() {
                .with_features(nodes[2].node.invoice_features())
                .with_route_hints(hop_hints);
        let (mut route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 10_000, 42);
-       assert_eq!(route.paths[0][1].short_channel_id, nodes[2].node.list_usable_channels()[0].inbound_scid_alias.unwrap());
+       assert_eq!(route.paths[0].hops[1].short_channel_id, nodes[2].node.list_usable_channels()[0].inbound_scid_alias.unwrap());
 
-       route.paths[0][1].fee_msat = 10_000_000; // Overshoot the last channel's value
+       route.paths[0].hops[1].fee_msat = 10_000_000; // Overshoot the last channel's value
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
@@ -509,11 +518,12 @@ fn test_scid_alias_returned() {
                PaymentFailedConditions::new().blamed_scid(last_hop[0].inbound_scid_alias.unwrap())
                        .blamed_chan_closed(false).expected_htlc_error_data(0x1000|7, &err_data));
 
-       route.paths[0][1].fee_msat = 10_000; // Reset to the correct payment amount
-       route.paths[0][0].fee_msat = 0; // But set fee paid to the middle hop to 0
+       route.paths[0].hops[1].fee_msat = 10_000; // Reset to the correct payment amount
+       route.paths[0].hops[0].fee_msat = 0; // But set fee paid to the middle hop to 0
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
@@ -599,6 +609,7 @@ fn test_0conf_channel_with_async_monitor() {
 
        let channel_id = funding_output.to_channel_id();
        nodes[1].chain_monitor.complete_sole_pending_chan_update(&channel_id);
+       expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        let bs_signed_locked = nodes[1].node.get_and_clear_pending_msg_events();
        assert_eq!(bs_signed_locked.len(), 2);
@@ -624,6 +635,22 @@ fn test_0conf_channel_with_async_monitor() {
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        nodes[0].chain_monitor.complete_sole_pending_chan_update(&channel_id);
+
+       let events = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 2);
+       match events[0] {
+               crate::events::Event::ChannelPending { ref counterparty_node_id, .. } => {
+                       assert_eq!(nodes[1].node.get_our_node_id(), *counterparty_node_id);
+               },
+               _ => panic!("Unexpected event"),
+       }
+       match events[1] {
+               crate::events::Event::ChannelReady { ref counterparty_node_id, .. } => {
+                       assert_eq!(nodes[1].node.get_our_node_id(), *counterparty_node_id);
+               },
+               _ => panic!("Unexpected event"),
+       }
+
        let as_locked_update = nodes[0].node.get_and_clear_pending_msg_events();
 
        // Note that the funding transaction is actually released when
@@ -638,7 +665,6 @@ fn test_0conf_channel_with_async_monitor() {
                }
                _ => panic!("Unexpected event"),
        }
-       expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
        expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        let bs_channel_update = get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[0].node.get_our_node_id());
@@ -666,7 +692,8 @@ fn test_0conf_channel_with_async_monitor() {
        // failure before we've ever confirmed the funding transaction. This previously caused a panic.
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let as_send = SendEvent::from_node(&nodes[0]);
@@ -812,7 +839,7 @@ fn test_0conf_channel_reorg() {
        assert_eq!(nodes[1].node.list_usable_channels()[0].short_channel_id.unwrap(), real_scid);
 
        let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 10_000);
-       assert_eq!(route.paths[0][0].short_channel_id, real_scid);
+       assert_eq!(route.paths[0].hops[0].short_channel_id, real_scid);
        send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1]]], 10_000, payment_hash, payment_secret);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 
index ffb6ba0e85a1340da52bc88d28168e05f0f5dfe1..67a48012789c1e30d6e56e519984f21eefb84eee 100644 (file)
@@ -15,7 +15,7 @@ use crate::chain::channelmonitor::ChannelMonitor;
 use crate::chain::keysinterface::EntropySource;
 use crate::chain::transaction::OutPoint;
 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
-use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId};
+use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
 use crate::util::enforcing_trait_impls::EnforcingSigner;
@@ -261,6 +261,9 @@ fn test_manager_serialize_deserialize_events() {
        }
        // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
 
+       expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
+       expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
+
        nodes.push(node_a);
        nodes.push(node_b);
 
@@ -603,7 +606,8 @@ fn test_forwardable_regen() {
 
        // First send a payment to nodes[1]
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -616,7 +620,8 @@ fn test_forwardable_regen() {
 
        // Next send a payment which is forwarded by nodes[1]
        let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
+               RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -694,11 +699,12 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
        assert_eq!(route.paths.len(), 2);
        route.paths.sort_by(|path_a, _| {
                // Sort the path so that the path through nodes[1] comes first
-               if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
+               if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
                        core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
        });
 
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2);
 
        // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
@@ -850,11 +856,12 @@ fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_ht
        let (mut route, payment_hash, payment_preimage, payment_secret) =
                get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
        if use_intercept {
-               route.paths[0][1].short_channel_id = intercept_scid;
+               route.paths[0].hops[1].short_channel_id = intercept_scid;
        }
        let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
        let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
+       nodes[0].node.send_payment_with_route(&route, payment_hash,
+               RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let payment_event = SendEvent::from_node(&nodes[0]);
@@ -941,7 +948,7 @@ fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_ht
                if claim_htlc {
                        confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
                } else {
-                       connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
+                       connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
                        let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
                        assert_eq!(bs_htlc_timeout_tx.len(), 1);
                        confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
index 6d89268cdc34357808178ec5a99d2330dcff61dd..e8f0c1259437142a31f3f3481e788fe031786ba9 100644 (file)
@@ -9,7 +9,7 @@
 
 //! Further functional tests which test blockchain reorganizations.
 
-use crate::chain::channelmonitor::ANTI_REORG_DELAY;
+use crate::chain::channelmonitor::{ANTI_REORG_DELAY, LATENCY_GRACE_PERIOD_BLOCKS};
 use crate::chain::transaction::OutPoint;
 use crate::chain::Confirm;
 use crate::events::{Event, MessageSendEventsProvider, ClosureReason, HTLCDestination};
@@ -103,7 +103,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
 
                // Give node 1 node 2's commitment transaction and get its response (timing the HTLC out)
                mine_transaction(&nodes[1], &node_2_commitment_txn[0]);
-               connect_blocks(&nodes[1], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
+               connect_blocks(&nodes[1], TEST_FINAL_CLTV); // Confirm blocks until the HTLC expires
                let node_1_commitment_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
                assert_eq!(node_1_commitment_txn.len(), 1); // ChannelMonitor: 1 offered HTLC-Timeout
                check_spends!(node_1_commitment_txn[0], node_2_commitment_txn[0]);
@@ -467,19 +467,21 @@ fn test_set_outpoints_partial_claiming() {
        }
 
        // Connect blocks on node B
-       connect_blocks(&nodes[1], 135);
+       connect_blocks(&nodes[1], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
        check_closed_broadcast!(nodes[1], true);
        check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        check_added_monitors!(nodes[1], 1);
        // Verify node B broadcast 2 HTLC-timeout txn
        let partial_claim_tx = {
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
+               let mut node_txn = nodes[1].tx_broadcaster.unique_txn_broadcast();
                assert_eq!(node_txn.len(), 3);
+               check_spends!(node_txn[0], chan.3);
                check_spends!(node_txn[1], node_txn[0]);
                check_spends!(node_txn[2], node_txn[0]);
                assert_eq!(node_txn[1].input.len(), 1);
                assert_eq!(node_txn[2].input.len(), 1);
-               node_txn[1].clone()
+               assert_ne!(node_txn[1].input[0].previous_output, node_txn[2].input[0].previous_output);
+               node_txn.remove(1)
        };
 
        // Broadcast partial claim on node A, should regenerate a claiming tx with HTLC dropped
index 6e7c0e12e043cb5d749bc51251af66b207c8afd7..f5be10698583f6860d41e7ee06218e8b51412b38 100644 (file)
@@ -12,7 +12,7 @@
 use crate::chain::keysinterface::{EntropySource, SignerProvider};
 use crate::chain::transaction::OutPoint;
 use crate::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
-use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId};
+use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId, RecipientOnionFields};
 use crate::routing::router::{PaymentParameters, get_route};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, ErrorAction};
@@ -98,8 +98,12 @@ fn updates_shutdown_wait() {
        let route_1 = get_route(&nodes[0].node.get_our_node_id(), &payment_params_1, &nodes[0].network_graph.read_only(), None, 100000, TEST_FINAL_CLTV, &logger, &scorer, &random_seed_bytes).unwrap();
        let payment_params_2 = PaymentParameters::from_node_id(nodes[0].node.get_our_node_id(), TEST_FINAL_CLTV).with_features(nodes[0].node.invoice_features());
        let route_2 = get_route(&nodes[1].node.get_our_node_id(), &payment_params_2, &nodes[1].network_graph.read_only(), None, 100000, TEST_FINAL_CLTV, &logger, &scorer, &random_seed_bytes).unwrap();
-       unwrap_send_err!(nodes[0].node.send_payment(&route_1, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable {..}, {});
-       unwrap_send_err!(nodes[1].node.send_payment(&route_2, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[0].node.send_payment_with_route(&route_1, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+               ), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[1].node.send_payment_with_route(&route_2, payment_hash,
+                       RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)
+               ), true, APIError::ChannelUnavailable {..}, {});
 
        nodes[2].node.claim_funds(payment_preimage_0);
        check_added_monitors!(nodes[2], 1);
@@ -159,7 +163,8 @@ fn htlc_fail_async_shutdown() {
        let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2);
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
+       nodes[0].node.send_payment_with_route(&route, our_payment_hash,
+               RecipientOnionFields::secret_only(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        assert_eq!(updates.update_add_htlcs.len(), 1);
index 49c03a443475cb0c0c4f6ed556f5a242cda7d40b..7dd2a99d150bfad018eab79dc3448402d0817fb7 100644 (file)
@@ -29,7 +29,7 @@
 //!
 //! # use lightning::ln::PaymentHash;
 //! # use lightning::offers::invoice::BlindedPayInfo;
-//! # use lightning::onion_message::BlindedPath;
+//! # use lightning::blinded_path::BlindedPath;
 //! #
 //! # fn create_payment_paths() -> Vec<(BlindedPath, BlindedPayInfo)> { unimplemented!() }
 //! # fn create_payment_hash() -> PaymentHash { unimplemented!() }
@@ -97,24 +97,27 @@ use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::hash_types::{WPubkeyHash, WScriptHash};
 use bitcoin::hashes::Hash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::{Message, PublicKey};
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, self};
 use bitcoin::secp256k1::schnorr::Signature;
 use bitcoin::util::address::{Address, Payload, WitnessVersion};
 use bitcoin::util::schnorr::TweakedPublicKey;
-use core::convert::TryFrom;
+use core::convert::{Infallible, TryFrom};
 use core::time::Duration;
 use crate::io;
+use crate::blinded_path::BlindedPath;
 use crate::ln::PaymentHash;
 use crate::ln::features::{BlindedHopFeatures, Bolt12InvoiceFeatures};
+use crate::ln::inbound_payment::ExpandedKey;
 use crate::ln::msgs::DecodeError;
-use crate::offers::invoice_request::{InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
-use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, WithoutSignatures, self};
-use crate::offers::offer::{Amount, OfferTlvStream, OfferTlvStreamRef};
+use crate::offers::invoice_request::{INVOICE_REQUEST_PAYER_ID_TYPE, INVOICE_REQUEST_TYPES, IV_BYTES as INVOICE_REQUEST_IV_BYTES, InvoiceRequest, InvoiceRequestContents, InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
+use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, TlvStream, WithoutSignatures, self};
+use crate::offers::offer::{Amount, OFFER_TYPES, OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{ParseError, ParsedMessage, SemanticError};
-use crate::offers::payer::{PayerTlvStream, PayerTlvStreamRef};
-use crate::offers::refund::{Refund, RefundContents};
-use crate::onion_message::BlindedPath;
+use crate::offers::payer::{PAYER_METADATA_TYPE, PayerTlvStream, PayerTlvStreamRef};
+use crate::offers::refund::{IV_BYTES as REFUND_IV_BYTES, Refund, RefundContents};
+use crate::offers::signer;
 use crate::util::ser::{HighZeroBytesDroppedBigSize, Iterable, SeekReadable, WithoutLength, Writeable, Writer};
+use crate::util::string::PrintableString;
 
 use crate::prelude::*;
 
@@ -123,7 +126,7 @@ use std::time::SystemTime;
 
 const DEFAULT_RELATIVE_EXPIRY: Duration = Duration::from_secs(7200);
 
-const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature");
+pub(super) const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature");
 
 /// Builds an [`Invoice`] from either:
 /// - an [`InvoiceRequest`] for the "offer to be paid" flow or
@@ -131,65 +134,141 @@ const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice", "signature")
 ///
 /// See [module-level documentation] for usage.
 ///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+///
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Refund`]: crate::offers::refund::Refund
 /// [module-level documentation]: self
-pub struct InvoiceBuilder<'a> {
+pub struct InvoiceBuilder<'a, S: SigningPubkeyStrategy> {
        invreq_bytes: &'a Vec<u8>,
        invoice: InvoiceContents,
+       keys: Option<KeyPair>,
+       signing_pubkey_strategy: core::marker::PhantomData<S>,
 }
 
-impl<'a> InvoiceBuilder<'a> {
+/// Indicates how [`Invoice::signing_pubkey`] was set.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub trait SigningPubkeyStrategy {}
+
+/// [`Invoice::signing_pubkey`] was explicitly set.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct ExplicitSigningPubkey {}
+
+/// [`Invoice::signing_pubkey`] was derived.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct DerivedSigningPubkey {}
+
+impl SigningPubkeyStrategy for ExplicitSigningPubkey {}
+impl SigningPubkeyStrategy for DerivedSigningPubkey {}
+
+impl<'a> InvoiceBuilder<'a, ExplicitSigningPubkey> {
        pub(super) fn for_offer(
                invoice_request: &'a InvoiceRequest, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>,
                created_at: Duration, payment_hash: PaymentHash
        ) -> Result<Self, SemanticError> {
-               let amount_msats = match invoice_request.amount_msats() {
-                       Some(amount_msats) => amount_msats,
-                       None => match invoice_request.contents.offer.amount() {
-                               Some(Amount::Bitcoin { amount_msats }) => {
-                                       amount_msats.checked_mul(invoice_request.quantity().unwrap_or(1))
-                                               .ok_or(SemanticError::InvalidAmount)?
-                               },
-                               Some(Amount::Currency { .. }) => return Err(SemanticError::UnsupportedCurrency),
-                               None => return Err(SemanticError::MissingAmount),
-                       },
-               };
-
+               let amount_msats = Self::check_amount_msats(invoice_request)?;
+               let signing_pubkey = invoice_request.contents.inner.offer.signing_pubkey();
                let contents = InvoiceContents::ForOffer {
                        invoice_request: invoice_request.contents.clone(),
-                       fields: InvoiceFields {
-                               payment_paths, created_at, relative_expiry: None, payment_hash, amount_msats,
-                               fallbacks: None, features: Bolt12InvoiceFeatures::empty(),
-                               signing_pubkey: invoice_request.contents.offer.signing_pubkey(),
-                       },
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
                };
 
-               Self::new(&invoice_request.bytes, contents)
+               Self::new(&invoice_request.bytes, contents, None)
        }
 
        pub(super) fn for_refund(
                refund: &'a Refund, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
                payment_hash: PaymentHash, signing_pubkey: PublicKey
        ) -> Result<Self, SemanticError> {
+               let amount_msats = refund.amount_msats();
                let contents = InvoiceContents::ForRefund {
                        refund: refund.contents.clone(),
-                       fields: InvoiceFields {
-                               payment_paths, created_at, relative_expiry: None, payment_hash,
-                               amount_msats: refund.amount_msats(), fallbacks: None,
-                               features: Bolt12InvoiceFeatures::empty(), signing_pubkey,
-                       },
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
                };
 
-               Self::new(&refund.bytes, contents)
+               Self::new(&refund.bytes, contents, None)
        }
+}
 
-       fn new(invreq_bytes: &'a Vec<u8>, contents: InvoiceContents) -> Result<Self, SemanticError> {
+impl<'a> InvoiceBuilder<'a, DerivedSigningPubkey> {
+       pub(super) fn for_offer_using_keys(
+               invoice_request: &'a InvoiceRequest, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>,
+               created_at: Duration, payment_hash: PaymentHash, keys: KeyPair
+       ) -> Result<Self, SemanticError> {
+               let amount_msats = Self::check_amount_msats(invoice_request)?;
+               let signing_pubkey = invoice_request.contents.inner.offer.signing_pubkey();
+               let contents = InvoiceContents::ForOffer {
+                       invoice_request: invoice_request.contents.clone(),
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
+               };
+
+               Self::new(&invoice_request.bytes, contents, Some(keys))
+       }
+
+       pub(super) fn for_refund_using_keys(
+               refund: &'a Refund, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
+               payment_hash: PaymentHash, keys: KeyPair,
+       ) -> Result<Self, SemanticError> {
+               let amount_msats = refund.amount_msats();
+               let signing_pubkey = keys.public_key();
+               let contents = InvoiceContents::ForRefund {
+                       refund: refund.contents.clone(),
+                       fields: Self::fields(
+                               payment_paths, created_at, payment_hash, amount_msats, signing_pubkey
+                       ),
+               };
+
+               Self::new(&refund.bytes, contents, Some(keys))
+       }
+}
+
+impl<'a, S: SigningPubkeyStrategy> InvoiceBuilder<'a, S> {
+       fn check_amount_msats(invoice_request: &InvoiceRequest) -> Result<u64, SemanticError> {
+               match invoice_request.amount_msats() {
+                       Some(amount_msats) => Ok(amount_msats),
+                       None => match invoice_request.contents.inner.offer.amount() {
+                               Some(Amount::Bitcoin { amount_msats }) => {
+                                       amount_msats.checked_mul(invoice_request.quantity().unwrap_or(1))
+                                               .ok_or(SemanticError::InvalidAmount)
+                               },
+                               Some(Amount::Currency { .. }) => Err(SemanticError::UnsupportedCurrency),
+                               None => Err(SemanticError::MissingAmount),
+                       },
+               }
+       }
+
+       fn fields(
+               payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, created_at: Duration,
+               payment_hash: PaymentHash, amount_msats: u64, signing_pubkey: PublicKey
+       ) -> InvoiceFields {
+               InvoiceFields {
+                       payment_paths, created_at, relative_expiry: None, payment_hash, amount_msats,
+                       fallbacks: None, features: Bolt12InvoiceFeatures::empty(), signing_pubkey,
+               }
+       }
+
+       fn new(
+               invreq_bytes: &'a Vec<u8>, contents: InvoiceContents, keys: Option<KeyPair>
+       ) -> Result<Self, SemanticError> {
                if contents.fields().payment_paths.is_empty() {
                        return Err(SemanticError::MissingPaths);
                }
 
-               Ok(Self { invreq_bytes, invoice: contents })
+               Ok(Self {
+                       invreq_bytes,
+                       invoice: contents,
+                       keys,
+                       signing_pubkey_strategy: core::marker::PhantomData,
+               })
        }
 
        /// Sets the [`Invoice::relative_expiry`] as seconds since [`Invoice::created_at`]. Any expiry
@@ -246,7 +325,9 @@ impl<'a> InvoiceBuilder<'a> {
                self.invoice.fields_mut().features.set_basic_mpp_optional();
                self
        }
+}
 
+impl<'a> InvoiceBuilder<'a, ExplicitSigningPubkey> {
        /// Builds an unsigned [`Invoice`] after checking for valid semantics. It can be signed by
        /// [`UnsignedInvoice::sign`].
        pub fn build(self) -> Result<UnsignedInvoice<'a>, SemanticError> {
@@ -256,11 +337,33 @@ impl<'a> InvoiceBuilder<'a> {
                        }
                }
 
-               let InvoiceBuilder { invreq_bytes, invoice } = self;
+               let InvoiceBuilder { invreq_bytes, invoice, .. } = self;
                Ok(UnsignedInvoice { invreq_bytes, invoice })
        }
 }
 
+impl<'a> InvoiceBuilder<'a, DerivedSigningPubkey> {
+       /// Builds a signed [`Invoice`] after checking for valid semantics.
+       pub fn build_and_sign<T: secp256k1::Signing>(
+               self, secp_ctx: &Secp256k1<T>
+       ) -> Result<Invoice, SemanticError> {
+               #[cfg(feature = "std")] {
+                       if self.invoice.is_offer_or_refund_expired() {
+                               return Err(SemanticError::AlreadyExpired);
+                       }
+               }
+
+               let InvoiceBuilder { invreq_bytes, invoice, keys, .. } = self;
+               let unsigned_invoice = UnsignedInvoice { invreq_bytes, invoice };
+
+               let keys = keys.unwrap();
+               let invoice = unsigned_invoice
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys)))
+                       .unwrap();
+               Ok(invoice)
+       }
+}
+
 /// A semantically valid [`Invoice`] that hasn't been signed.
 pub struct UnsignedInvoice<'a> {
        invreq_bytes: &'a Vec<u8>,
@@ -274,6 +377,8 @@ impl<'a> UnsignedInvoice<'a> {
        }
 
        /// Signs the invoice using the given function.
+       ///
+       /// This is not exported to bindings users as functions aren't currently mapped.
        pub fn sign<F, E>(self, sign: F) -> Result<Invoice, SignError<E>>
        where
                F: FnOnce(&Message) -> Result<Signature, E>
@@ -310,10 +415,13 @@ impl<'a> UnsignedInvoice<'a> {
 /// An invoice may be sent in response to an [`InvoiceRequest`] in the case of an offer or sent
 /// directly after scanning a refund. It includes all the information needed to pay a recipient.
 ///
+/// This is not exported to bindings users as its name conflicts with the BOLT 11 Invoice type.
+///
 /// [`Offer`]: crate::offers::offer::Offer
 /// [`Refund`]: crate::offers::refund::Refund
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Invoice {
        bytes: Vec<u8>,
        contents: InvoiceContents,
@@ -324,7 +432,8 @@ pub struct Invoice {
 ///
 /// [`Offer`]: crate::offers::offer::Offer
 /// [`Refund`]: crate::offers::refund::Refund
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 enum InvoiceContents {
        /// Contents for an [`Invoice`] corresponding to an [`Offer`].
        ///
@@ -356,6 +465,12 @@ struct InvoiceFields {
 }
 
 impl Invoice {
+       /// A complete description of the purpose of the originating offer or refund. Intended to be
+       /// displayed to the user but with the caveat that it has not been verified in any way.
+       pub fn description(&self) -> PrintableString {
+               self.contents.description()
+       }
+
        /// Paths to the recipient originating from publicly reachable nodes, including information
        /// needed for routing payments across them.
        ///
@@ -469,8 +584,20 @@ impl Invoice {
                self.signature
        }
 
+       /// Hash that was used for signing the invoice.
+       pub fn signable_hash(&self) -> [u8; 32] {
+               merkle::message_digest(SIGNATURE_TAG, &self.bytes).as_ref().clone()
+       }
+
+       /// Verifies that the invoice was for a request or refund created using the given key.
+       pub fn verify<T: secp256k1::Signing>(
+               &self, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> bool {
+               self.contents.verify(TlvStream::new(&self.bytes), key, secp_ctx)
+       }
+
        #[cfg(test)]
-       fn as_tlv_stream(&self) -> FullInvoiceTlvStreamRef {
+       pub(super) fn as_tlv_stream(&self) -> FullInvoiceTlvStreamRef {
                let (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream) =
                        self.contents.as_tlv_stream();
                let signature_tlv_stream = SignatureTlvStreamRef {
@@ -486,7 +613,8 @@ impl InvoiceContents {
        #[cfg(feature = "std")]
        fn is_offer_or_refund_expired(&self) -> bool {
                match self {
-                       InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.offer.is_expired(),
+                       InvoiceContents::ForOffer { invoice_request, .. } =>
+                               invoice_request.inner.offer.is_expired(),
                        InvoiceContents::ForRefund { refund, .. } => refund.is_expired(),
                }
        }
@@ -498,6 +626,15 @@ impl InvoiceContents {
                }
        }
 
+       fn description(&self) -> PrintableString {
+               match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => {
+                               invoice_request.inner.offer.description()
+                       },
+                       InvoiceContents::ForRefund { refund, .. } => refund.description(),
+               }
+       }
+
        fn fields(&self) -> &InvoiceFields {
                match self {
                        InvoiceContents::ForOffer { fields, .. } => fields,
@@ -512,6 +649,41 @@ impl InvoiceContents {
                }
        }
 
+       fn verify<T: secp256k1::Signing>(
+               &self, tlv_stream: TlvStream<'_>, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> bool {
+               let offer_records = tlv_stream.clone().range(OFFER_TYPES);
+               let invreq_records = tlv_stream.range(INVOICE_REQUEST_TYPES).filter(|record| {
+                       match record.r#type {
+                               PAYER_METADATA_TYPE => false, // Should be outside range
+                               INVOICE_REQUEST_PAYER_ID_TYPE => !self.derives_keys(),
+                               _ => true,
+                       }
+               });
+               let tlv_stream = offer_records.chain(invreq_records);
+
+               let (metadata, payer_id, iv_bytes) = match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => {
+                               (invoice_request.metadata(), invoice_request.payer_id(), INVOICE_REQUEST_IV_BYTES)
+                       },
+                       InvoiceContents::ForRefund { refund, .. } => {
+                               (refund.metadata(), refund.payer_id(), REFUND_IV_BYTES)
+                       },
+               };
+
+               match signer::verify_metadata(metadata, key, iv_bytes, payer_id, tlv_stream, secp_ctx) {
+                       Ok(_) => true,
+                       Err(()) => false,
+               }
+       }
+
+       fn derives_keys(&self) -> bool {
+               match self {
+                       InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.derives_keys(),
+                       InvoiceContents::ForRefund { refund, .. } => refund.derives_keys(),
+               }
+       }
+
        fn as_tlv_stream(&self) -> PartialInvoiceTlvStreamRef {
                let (payer, offer, invoice_request) = match self {
                        InvoiceContents::ForOffer { invoice_request, .. } => invoice_request.as_tlv_stream(),
@@ -588,7 +760,7 @@ type BlindedPayInfoIter<'a> = core::iter::Map<
 >;
 
 /// Information needed to route a payment across a [`BlindedPath`].
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, Hash, Eq, PartialEq)]
 pub struct BlindedPayInfo {
        /// Base fee charged (in millisatoshi) for the entire blinded path.
        pub fee_base_msat: u32,
@@ -772,67 +944,30 @@ impl TryFrom<PartialInvoiceTlvStream> for InvoiceContents {
 
 #[cfg(test)]
 mod tests {
-       use super::{DEFAULT_RELATIVE_EXPIRY, BlindedPayInfo, FallbackAddress, FullInvoiceTlvStreamRef, Invoice, InvoiceTlvStreamRef, SIGNATURE_TAG};
+       use super::{DEFAULT_RELATIVE_EXPIRY, FallbackAddress, FullInvoiceTlvStreamRef, Invoice, InvoiceTlvStreamRef, SIGNATURE_TAG};
 
        use bitcoin::blockdata::script::Script;
        use bitcoin::hashes::Hash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey, XOnlyPublicKey, self};
-       use bitcoin::secp256k1::schnorr::Signature;
+       use bitcoin::secp256k1::{Message, Secp256k1, XOnlyPublicKey, self};
        use bitcoin::util::address::{Address, Payload, WitnessVersion};
        use bitcoin::util::schnorr::TweakedPublicKey;
-       use core::convert::{Infallible, TryFrom};
+       use core::convert::TryFrom;
        use core::time::Duration;
-       use crate::ln::PaymentHash;
+       use crate::blinded_path::{BlindedHop, BlindedPath};
+       use crate::chain::keysinterface::KeyMaterial;
+       use crate::ln::features::Bolt12InvoiceFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::DecodeError;
-       use crate::ln::features::{BlindedHopFeatures, Bolt12InvoiceFeatures};
        use crate::offers::invoice_request::InvoiceRequestTlvStreamRef;
        use crate::offers::merkle::{SignError, SignatureTlvStreamRef, self};
        use crate::offers::offer::{OfferBuilder, OfferTlvStreamRef, Quantity};
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
        use crate::offers::refund::RefundBuilder;
-       use crate::onion_message::{BlindedHop, BlindedPath};
+       use crate::offers::test_utils::*;
        use crate::util::ser::{BigSize, Iterable, Writeable};
-
-       fn payer_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
-       }
-
-       fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn payer_pubkey() -> PublicKey {
-               payer_keys().public_key()
-       }
-
-       fn recipient_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap())
-       }
-
-       fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn recipient_pubkey() -> PublicKey {
-               recipient_keys().public_key()
-       }
-
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
+       use crate::util::string::PrintableString;
 
        trait ToBytes {
                fn to_bytes(&self) -> Vec<u8>;
@@ -850,58 +985,6 @@ mod tests {
                }
        }
 
-       fn payment_paths() -> Vec<(BlindedPath, BlindedPayInfo)> {
-               let paths = vec![
-                       BlindedPath {
-                               introduction_node_id: pubkey(40),
-                               blinding_point: pubkey(41),
-                               blinded_hops: vec![
-                                       BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
-                                       BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
-                               ],
-                       },
-                       BlindedPath {
-                               introduction_node_id: pubkey(40),
-                               blinding_point: pubkey(41),
-                               blinded_hops: vec![
-                                       BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
-                                       BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
-                               ],
-                       },
-               ];
-
-               let payinfo = vec![
-                       BlindedPayInfo {
-                               fee_base_msat: 1,
-                               fee_proportional_millionths: 1_000,
-                               cltv_expiry_delta: 42,
-                               htlc_minimum_msat: 100,
-                               htlc_maximum_msat: 1_000_000_000_000,
-                               features: BlindedHopFeatures::empty(),
-                       },
-                       BlindedPayInfo {
-                               fee_base_msat: 1,
-                               fee_proportional_millionths: 1_000,
-                               cltv_expiry_delta: 42,
-                               htlc_minimum_msat: 100,
-                               htlc_maximum_msat: 1_000_000_000_000,
-                               features: BlindedHopFeatures::empty(),
-                       },
-               ];
-
-               paths.into_iter().zip(payinfo.into_iter()).collect()
-       }
-
-       fn payment_hash() -> PaymentHash {
-               PaymentHash([42; 32])
-       }
-
-       fn now() -> Duration {
-               std::time::SystemTime::now()
-                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
-                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH")
-       }
-
        #[test]
        fn builds_invoice_for_offer_with_defaults() {
                let payment_paths = payment_paths();
@@ -921,6 +1004,7 @@ mod tests {
                invoice.write(&mut buffer).unwrap();
 
                assert_eq!(invoice.bytes, buffer.as_slice());
+               assert_eq!(invoice.description(), PrintableString("foo"));
                assert_eq!(invoice.payment_paths(), payment_paths.as_slice());
                assert_eq!(invoice.created_at(), now);
                assert_eq!(invoice.relative_expiry(), DEFAULT_RELATIVE_EXPIRY);
@@ -937,6 +1021,11 @@ mod tests {
                        ).is_ok()
                );
 
+               let digest = Message::from_slice(&invoice.signable_hash()).unwrap();
+               let pubkey = recipient_pubkey().into();
+               let secp_ctx = Secp256k1::verification_only();
+               assert!(secp_ctx.verify_schnorr(&invoice.signature, &digest, &pubkey).is_ok());
+
                assert_eq!(
                        invoice.as_tlv_stream(),
                        (
@@ -998,6 +1087,7 @@ mod tests {
                invoice.write(&mut buffer).unwrap();
 
                assert_eq!(invoice.bytes, buffer.as_slice());
+               assert_eq!(invoice.description(), PrintableString("foo"));
                assert_eq!(invoice.payment_paths(), payment_paths.as_slice());
                assert_eq!(invoice.created_at(), now);
                assert_eq!(invoice.relative_expiry(), DEFAULT_RELATIVE_EXPIRY);
@@ -1123,6 +1213,87 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_invoice_from_offer_using_derived_keys() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(42), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .path(blinded_path)
+                       .build().unwrap();
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+
+               if let Err(e) = invoice_request
+                       .verify_and_respond_using_derived_keys_no_std(
+                               payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+                       )
+                       .unwrap()
+                       .build_and_sign(&secp_ctx)
+               {
+                       panic!("error building invoice: {:?}", e);
+               }
+
+               let expanded_key = ExpandedKey::new(&KeyMaterial([41; 32]));
+               match invoice_request.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+               ) {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::InvalidMetadata),
+               }
+
+               let desc = "foo".to_string();
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+
+               match invoice_request.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths(), payment_hash(), now(), &expanded_key, &secp_ctx
+               ) {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::InvalidMetadata),
+               }
+       }
+
+       #[test]
+       fn builds_invoice_from_refund_using_derived_keys() {
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let refund = RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
+                       .build().unwrap();
+
+               if let Err(e) = refund
+                       .respond_using_derived_keys_no_std(
+                               payment_paths(), payment_hash(), now(), &expanded_key, &entropy
+                       )
+                       .unwrap()
+                       .build_and_sign(&secp_ctx)
+               {
+                       panic!("error building invoice: {:?}", e);
+               }
+       }
+
        #[test]
        fn builds_invoice_with_relative_expiry() {
                let now = now();
index a1a0520c62259409b4cd516ed9607eb17bad3296..8d4755537078f5a8698d8ef7d87fb71ca168206d 100644 (file)
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::{Message, PublicKey};
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, self};
 use bitcoin::secp256k1::schnorr::Signature;
-use core::convert::TryFrom;
+use core::convert::{Infallible, TryFrom};
+use core::ops::Deref;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
+use crate::blinded_path::BlindedPath;
 use crate::ln::PaymentHash;
 use crate::ln::features::InvoiceRequestFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::DecodeError;
-use crate::offers::invoice::{BlindedPayInfo, InvoiceBuilder};
+use crate::offers::invoice::{BlindedPayInfo, DerivedSigningPubkey, ExplicitSigningPubkey, InvoiceBuilder};
 use crate::offers::merkle::{SignError, SignatureTlvStream, SignatureTlvStreamRef, self};
 use crate::offers::offer::{Offer, OfferContents, OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{ParseError, ParsedMessage, SemanticError};
 use crate::offers::payer::{PayerContents, PayerTlvStream, PayerTlvStreamRef};
-use crate::onion_message::BlindedPath;
+use crate::offers::signer::{Metadata, MetadataMaterial};
 use crate::util::ser::{HighZeroBytesDroppedBigSize, SeekReadable, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
 
@@ -74,25 +78,91 @@ use crate::prelude::*;
 
 const SIGNATURE_TAG: &'static str = concat!("lightning", "invoice_request", "signature");
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Invreq ~~~~~";
+
 /// Builds an [`InvoiceRequest`] from an [`Offer`] for the "offer to be paid" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+///
 /// [module-level documentation]: self
-pub struct InvoiceRequestBuilder<'a> {
+pub struct InvoiceRequestBuilder<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> {
        offer: &'a Offer,
-       invoice_request: InvoiceRequestContents,
+       invoice_request: InvoiceRequestContentsWithoutPayerId,
+       payer_id: Option<PublicKey>,
+       payer_id_strategy: core::marker::PhantomData<P>,
+       secp_ctx: Option<&'b Secp256k1<T>>,
 }
 
-impl<'a> InvoiceRequestBuilder<'a> {
+/// Indicates how [`InvoiceRequest::payer_id`] will be set.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub trait PayerIdStrategy {}
+
+/// [`InvoiceRequest::payer_id`] will be explicitly set.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct ExplicitPayerId {}
+
+/// [`InvoiceRequest::payer_id`] will be derived.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct DerivedPayerId {}
+
+impl PayerIdStrategy for ExplicitPayerId {}
+impl PayerIdStrategy for DerivedPayerId {}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, ExplicitPayerId, T> {
        pub(super) fn new(offer: &'a Offer, metadata: Vec<u8>, payer_id: PublicKey) -> Self {
                Self {
                        offer,
-                       invoice_request: InvoiceRequestContents {
-                               payer: PayerContents(metadata), offer: offer.contents.clone(), chain: None,
-                               amount_msats: None, features: InvoiceRequestFeatures::empty(), quantity: None,
-                               payer_id, payer_note: None,
-                       },
+                       invoice_request: Self::create_contents(offer, Metadata::Bytes(metadata)),
+                       payer_id: Some(payer_id),
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+
+       pub(super) fn deriving_metadata<ES: Deref>(
+               offer: &'a Offer, payer_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::Derived(derivation_material);
+               Self {
+                       offer,
+                       invoice_request: Self::create_contents(offer, metadata),
+                       payer_id: Some(payer_id),
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T> {
+       pub(super) fn deriving_payer_id<ES: Deref>(
+               offer: &'a Offer, expanded_key: &ExpandedKey, entropy_source: ES, secp_ctx: &'b Secp256k1<T>
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               Self {
+                       offer,
+                       invoice_request: Self::create_contents(offer, metadata),
+                       payer_id: None,
+                       payer_id_strategy: core::marker::PhantomData,
+                       secp_ctx: Some(secp_ctx),
+               }
+       }
+}
+
+impl<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, P, T> {
+       fn create_contents(offer: &Offer, metadata: Metadata) -> InvoiceRequestContentsWithoutPayerId {
+               let offer = offer.contents.clone();
+               InvoiceRequestContentsWithoutPayerId {
+                       payer: PayerContents(metadata), offer, chain: None, amount_msats: None,
+                       features: InvoiceRequestFeatures::empty(), quantity: None, payer_note: None,
                }
        }
 
@@ -143,9 +213,10 @@ impl<'a> InvoiceRequestBuilder<'a> {
                self
        }
 
-       /// Builds an unsigned [`InvoiceRequest`] after checking for valid semantics. It can be signed
-       /// by [`UnsignedInvoiceRequest::sign`].
-       pub fn build(mut self) -> Result<UnsignedInvoiceRequest<'a>, SemanticError> {
+       fn build_with_checks(mut self) -> Result<
+               (UnsignedInvoiceRequest<'a>, Option<KeyPair>, Option<&'b Secp256k1<T>>),
+               SemanticError
+       > {
                #[cfg(feature = "std")] {
                        if self.offer.is_expired() {
                                return Err(SemanticError::AlreadyExpired);
@@ -170,13 +241,79 @@ impl<'a> InvoiceRequestBuilder<'a> {
                        self.invoice_request.amount_msats, self.invoice_request.quantity
                )?;
 
-               let InvoiceRequestBuilder { offer, invoice_request } = self;
-               Ok(UnsignedInvoiceRequest { offer, invoice_request })
+               Ok(self.build_without_checks())
+       }
+
+       fn build_without_checks(mut self) ->
+               (UnsignedInvoiceRequest<'a>, Option<KeyPair>, Option<&'b Secp256k1<T>>)
+       {
+               // Create the metadata for stateless verification of an Invoice.
+               let mut keys = None;
+               let secp_ctx = self.secp_ctx.clone();
+               if self.invoice_request.payer.0.has_derivation_material() {
+                       let mut metadata = core::mem::take(&mut self.invoice_request.payer.0);
+
+                       let mut tlv_stream = self.invoice_request.as_tlv_stream();
+                       debug_assert!(tlv_stream.2.payer_id.is_none());
+                       tlv_stream.0.metadata = None;
+                       if !metadata.derives_keys() {
+                               tlv_stream.2.payer_id = self.payer_id.as_ref();
+                       }
+
+                       let (derived_metadata, derived_keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                       metadata = derived_metadata;
+                       keys = derived_keys;
+                       if let Some(keys) = keys {
+                               debug_assert!(self.payer_id.is_none());
+                               self.payer_id = Some(keys.public_key());
+                       }
+
+                       self.invoice_request.payer.0 = metadata;
+               }
+
+               debug_assert!(self.invoice_request.payer.0.as_bytes().is_some());
+               debug_assert!(self.payer_id.is_some());
+               let payer_id = self.payer_id.unwrap();
+
+               let unsigned_invoice = UnsignedInvoiceRequest {
+                       offer: self.offer,
+                       invoice_request: InvoiceRequestContents {
+                               inner: self.invoice_request,
+                               payer_id,
+                       },
+               };
+
+               (unsigned_invoice, keys, secp_ctx)
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, ExplicitPayerId, T> {
+       /// Builds an unsigned [`InvoiceRequest`] after checking for valid semantics. It can be signed
+       /// by [`UnsignedInvoiceRequest::sign`].
+       pub fn build(self) -> Result<UnsignedInvoiceRequest<'a>, SemanticError> {
+               let (unsigned_invoice_request, keys, _) = self.build_with_checks()?;
+               debug_assert!(keys.is_none());
+               Ok(unsigned_invoice_request)
+       }
+}
+
+impl<'a, 'b, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T> {
+       /// Builds a signed [`InvoiceRequest`] after checking for valid semantics.
+       pub fn build_and_sign(self) -> Result<InvoiceRequest, SemanticError> {
+               let (unsigned_invoice_request, keys, secp_ctx) = self.build_with_checks()?;
+               debug_assert!(keys.is_some());
+
+               let secp_ctx = secp_ctx.unwrap();
+               let keys = keys.unwrap();
+               let invoice_request = unsigned_invoice_request
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys)))
+                       .unwrap();
+               Ok(invoice_request)
        }
 }
 
 #[cfg(test)]
-impl<'a> InvoiceRequestBuilder<'a> {
+impl<'a, 'b, P: PayerIdStrategy, T: secp256k1::Signing> InvoiceRequestBuilder<'a, 'b, P, T> {
        fn chain_unchecked(mut self, network: Network) -> Self {
                let chain = ChainHash::using_genesis_block(network);
                self.invoice_request.chain = Some(chain);
@@ -199,8 +336,7 @@ impl<'a> InvoiceRequestBuilder<'a> {
        }
 
        pub(super) fn build_unchecked(self) -> UnsignedInvoiceRequest<'a> {
-               let InvoiceRequestBuilder { offer, invoice_request } = self;
-               UnsignedInvoiceRequest { offer, invoice_request }
+               self.build_without_checks().0
        }
 }
 
@@ -212,6 +348,8 @@ pub struct UnsignedInvoiceRequest<'a> {
 
 impl<'a> UnsignedInvoiceRequest<'a> {
        /// Signs the invoice request using the given function.
+       ///
+       /// This is not exported to bindings users as functions are not yet mapped.
        pub fn sign<F, E>(self, sign: F) -> Result<InvoiceRequest, SignError<E>>
        where
                F: FnOnce(&Message) -> Result<Signature, E>
@@ -250,7 +388,8 @@ impl<'a> UnsignedInvoiceRequest<'a> {
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
 /// [`Offer`]: crate::offers::offer::Offer
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct InvoiceRequest {
        pub(super) bytes: Vec<u8>,
        pub(super) contents: InvoiceRequestContents,
@@ -260,15 +399,22 @@ pub struct InvoiceRequest {
 /// The contents of an [`InvoiceRequest`], which may be shared with an [`Invoice`].
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct InvoiceRequestContents {
+       pub(super) inner: InvoiceRequestContentsWithoutPayerId,
+       payer_id: PublicKey,
+}
+
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub(super) struct InvoiceRequestContentsWithoutPayerId {
        payer: PayerContents,
        pub(super) offer: OfferContents,
        chain: Option<ChainHash>,
        amount_msats: Option<u64>,
        features: InvoiceRequestFeatures,
        quantity: Option<u64>,
-       payer_id: PublicKey,
        payer_note: Option<String>,
 }
 
@@ -278,7 +424,7 @@ impl InvoiceRequest {
        ///
        /// [`payer_id`]: Self::payer_id
        pub fn metadata(&self) -> &[u8] {
-               &self.contents.payer.0[..]
+               self.contents.metadata()
        }
 
        /// A chain from [`Offer::chains`] that the offer is valid for.
@@ -291,17 +437,17 @@ impl InvoiceRequest {
        ///
        /// [`chain`]: Self::chain
        pub fn amount_msats(&self) -> Option<u64> {
-               self.contents.amount_msats
+               self.contents.inner.amount_msats
        }
 
        /// Features pertaining to requesting an invoice.
        pub fn features(&self) -> &InvoiceRequestFeatures {
-               &self.contents.features
+               &self.contents.inner.features
        }
 
        /// The quantity of the offer's item conforming to [`Offer::is_valid_quantity`].
        pub fn quantity(&self) -> Option<u64> {
-               self.contents.quantity
+               self.contents.inner.quantity
        }
 
        /// A possibly transient pubkey used to sign the invoice request.
@@ -312,7 +458,8 @@ impl InvoiceRequest {
        /// A payer-provided note which will be seen by the recipient and reflected back in the invoice
        /// response.
        pub fn payer_note(&self) -> Option<PrintableString> {
-               self.contents.payer_note.as_ref().map(|payer_note| PrintableString(payer_note.as_str()))
+               self.contents.inner.payer_note.as_ref()
+                       .map(|payer_note| PrintableString(payer_note.as_str()))
        }
 
        /// Signature of the invoice request using [`payer_id`].
@@ -322,18 +469,19 @@ impl InvoiceRequest {
                self.signature
        }
 
-       /// Creates an [`Invoice`] for the request with the given required fields and using the
+       /// Creates an [`InvoiceBuilder`] for the request with the given required fields and using the
        /// [`Duration`] since [`std::time::SystemTime::UNIX_EPOCH`] as the creation time.
        ///
        /// See [`InvoiceRequest::respond_with_no_std`] for further details where the aforementioned
        /// creation time is used for the `created_at` parameter.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
        /// [`Duration`]: core::time::Duration
        #[cfg(feature = "std")]
        pub fn respond_with(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                let created_at = std::time::SystemTime::now()
                        .duration_since(std::time::SystemTime::UNIX_EPOCH)
                        .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
@@ -341,7 +489,7 @@ impl InvoiceRequest {
                self.respond_with_no_std(payment_paths, payment_hash, created_at)
        }
 
-       /// Creates an [`Invoice`] for the request with the given required fields.
+       /// Creates an [`InvoiceBuilder`] for the request with the given required fields.
        ///
        /// Unless [`InvoiceBuilder::relative_expiry`] is set, the invoice will expire two hours after
        /// `created_at`, which is used to set [`Invoice::created_at`]. Useful for `no-std` builds where
@@ -357,12 +505,13 @@ impl InvoiceRequest {
        ///
        /// Errors if the request contains unknown required features.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
        /// [`Invoice::created_at`]: crate::offers::invoice::Invoice::created_at
        pub fn respond_with_no_std(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                created_at: core::time::Duration
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -370,6 +519,66 @@ impl InvoiceRequest {
                InvoiceBuilder::for_offer(self, payment_paths, created_at, payment_hash)
        }
 
+       /// Creates an [`InvoiceBuilder`] for the request using the given required fields and that uses
+       /// derived signing keys from the originating [`Offer`] to sign the [`Invoice`]. Must use the
+       /// same [`ExpandedKey`] as the one used to create the offer.
+       ///
+       /// See [`InvoiceRequest::respond_with`] for further details.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       #[cfg(feature = "std")]
+       pub fn verify_and_respond_using_derived_keys<T: secp256k1::Signing>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               expanded_key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError> {
+               let created_at = std::time::SystemTime::now()
+                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
+                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
+
+               self.verify_and_respond_using_derived_keys_no_std(
+                       payment_paths, payment_hash, created_at, expanded_key, secp_ctx
+               )
+       }
+
+       /// Creates an [`InvoiceBuilder`] for the request using the given required fields and that uses
+       /// derived signing keys from the originating [`Offer`] to sign the [`Invoice`]. Must use the
+       /// same [`ExpandedKey`] as the one used to create the offer.
+       ///
+       /// See [`InvoiceRequest::respond_with_no_std`] for further details.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn verify_and_respond_using_derived_keys_no_std<T: secp256k1::Signing>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               created_at: core::time::Duration, expanded_key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError> {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               let keys = match self.verify(expanded_key, secp_ctx) {
+                       Err(()) => return Err(SemanticError::InvalidMetadata),
+                       Ok(None) => return Err(SemanticError::InvalidMetadata),
+                       Ok(Some(keys)) => keys,
+               };
+
+               InvoiceBuilder::for_offer_using_keys(self, payment_paths, created_at, payment_hash, keys)
+       }
+
+       /// Verifies that the request was for an offer created using the given key. Returns the derived
+       /// keys need to sign an [`Invoice`] for the request if they could be extracted from the
+       /// metadata.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn verify<T: secp256k1::Signing>(
+               &self, key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<Option<KeyPair>, ()> {
+               self.contents.inner.offer.verify(&self.bytes, key, secp_ctx)
+       }
+
        #[cfg(test)]
        fn as_tlv_stream(&self) -> FullInvoiceRequestTlvStreamRef {
                let (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream) =
@@ -382,13 +591,41 @@ impl InvoiceRequest {
 }
 
 impl InvoiceRequestContents {
+       pub fn metadata(&self) -> &[u8] {
+               self.inner.metadata()
+       }
+
+       pub(super) fn derives_keys(&self) -> bool {
+               self.inner.payer.0.derives_keys()
+       }
+
+       pub(super) fn chain(&self) -> ChainHash {
+               self.inner.chain()
+       }
+
+       pub(super) fn payer_id(&self) -> PublicKey {
+               self.payer_id
+       }
+
+       pub(super) fn as_tlv_stream(&self) -> PartialInvoiceRequestTlvStreamRef {
+               let (payer, offer, mut invoice_request) = self.inner.as_tlv_stream();
+               invoice_request.payer_id = Some(&self.payer_id);
+               (payer, offer, invoice_request)
+       }
+}
+
+impl InvoiceRequestContentsWithoutPayerId {
+       pub(super) fn metadata(&self) -> &[u8] {
+               self.payer.0.as_bytes().map(|bytes| bytes.as_slice()).unwrap_or(&[])
+       }
+
        pub(super) fn chain(&self) -> ChainHash {
                self.chain.unwrap_or_else(|| self.offer.implied_chain())
        }
 
        pub(super) fn as_tlv_stream(&self) -> PartialInvoiceRequestTlvStreamRef {
                let payer = PayerTlvStreamRef {
-                       metadata: Some(&self.payer.0),
+                       metadata: self.payer.0.as_bytes(),
                };
 
                let offer = self.offer.as_tlv_stream();
@@ -403,7 +640,7 @@ impl InvoiceRequestContents {
                        amount: self.amount_msats,
                        features,
                        quantity: self.quantity,
-                       payer_id: Some(&self.payer_id),
+                       payer_id: None,
                        payer_note: self.payer_note.as_ref(),
                };
 
@@ -423,12 +660,20 @@ impl Writeable for InvoiceRequestContents {
        }
 }
 
-tlv_stream!(InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef, 80..160, {
+/// Valid type range for invoice_request TLV records.
+pub(super) const INVOICE_REQUEST_TYPES: core::ops::Range<u64> = 80..160;
+
+/// TLV record type for [`InvoiceRequest::payer_id`] and [`Refund::payer_id`].
+///
+/// [`Refund::payer_id`]: crate::offers::refund::Refund::payer_id
+pub(super) const INVOICE_REQUEST_PAYER_ID_TYPE: u64 = 88;
+
+tlv_stream!(InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef, INVOICE_REQUEST_TYPES, {
        (80, chain: ChainHash),
        (82, amount: (u64, HighZeroBytesDroppedBigSize)),
        (84, features: (InvoiceRequestFeatures, WithoutLength)),
        (86, quantity: (u64, HighZeroBytesDroppedBigSize)),
-       (88, payer_id: PublicKey),
+       (INVOICE_REQUEST_PAYER_ID_TYPE, payer_id: PublicKey),
        (89, payer_note: (String, WithoutLength)),
 });
 
@@ -498,7 +743,7 @@ impl TryFrom<PartialInvoiceRequestTlvStream> for InvoiceRequestContents {
 
                let payer = match metadata {
                        None => return Err(SemanticError::MissingPayerMetadata),
-                       Some(metadata) => PayerContents(metadata),
+                       Some(metadata) => PayerContents(Metadata::Bytes(metadata)),
                };
                let offer = OfferContents::try_from(offer_tlv_stream)?;
 
@@ -521,7 +766,10 @@ impl TryFrom<PartialInvoiceRequestTlvStream> for InvoiceRequestContents {
                };
 
                Ok(InvoiceRequestContents {
-                       payer, offer, chain, amount_msats: amount, features, quantity, payer_id, payer_note,
+                       inner: InvoiceRequestContentsWithoutPayerId {
+                               payer, offer, chain, amount_msats: amount, features, quantity, payer_note,
+                       },
+                       payer_id,
                })
        }
 }
@@ -532,47 +780,24 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey, self};
-       use bitcoin::secp256k1::schnorr::Signature;
+       use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey, self};
        use core::convert::{Infallible, TryFrom};
        use core::num::NonZeroU64;
        #[cfg(feature = "std")]
        use core::time::Duration;
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::InvoiceRequestFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
+       use crate::offers::invoice::{Invoice, SIGNATURE_TAG as INVOICE_SIGNATURE_TAG};
        use crate::offers::merkle::{SignError, SignatureTlvStreamRef, self};
        use crate::offers::offer::{Amount, OfferBuilder, OfferTlvStreamRef, Quantity};
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
+       use crate::offers::test_utils::*;
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn payer_keys() -> KeyPair {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
-       }
-
-       fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn payer_pubkey() -> PublicKey {
-               payer_keys().public_key()
-       }
-
-       fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
-               let secp_ctx = Secp256k1::new();
-               let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
-               Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
-       }
-
-       fn recipient_pubkey() -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap()).public_key()
-       }
-
        #[test]
        fn builds_invoice_request_with_defaults() {
                let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey())
@@ -661,6 +886,148 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_invoice_request_with_derived_metadata() {
+               let payer_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer
+                       .request_invoice_deriving_metadata(payer_id, &expanded_key, &entropy)
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert_eq!(invoice_request.payer_id(), payer_pubkey());
+
+               let invoice = invoice_request.respond_with_no_std(payment_paths(), payment_hash(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream,
+                       mut invoice_tlv_stream, mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               invoice_request_tlv_stream.amount = Some(2000);
+               invoice_tlv_stream.amount = Some(2000);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered metadata
+               let (
+                       mut payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream,
+                       mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               let metadata = payer_tlv_stream.metadata.unwrap().iter().copied().rev().collect();
+               payer_tlv_stream.metadata = Some(&metadata);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
+       #[test]
+       fn builds_invoice_request_with_derived_payer_id() {
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap();
+               let invoice_request = offer
+                       .request_invoice_deriving_payer_id(&expanded_key, &entropy, &secp_ctx)
+                       .unwrap()
+                       .build_and_sign()
+                       .unwrap();
+
+               let invoice = invoice_request.respond_with_no_std(payment_paths(), payment_hash(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream,
+                       mut invoice_tlv_stream, mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               invoice_request_tlv_stream.amount = Some(2000);
+               invoice_tlv_stream.amount = Some(2000);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered payer id
+               let (
+                       payer_tlv_stream, offer_tlv_stream, mut invoice_request_tlv_stream, invoice_tlv_stream,
+                       mut signature_tlv_stream
+               ) = invoice.as_tlv_stream();
+               let payer_id = pubkey(1);
+               invoice_request_tlv_stream.payer_id = Some(&payer_id);
+
+               let tlv_stream =
+                       (payer_tlv_stream, offer_tlv_stream, invoice_request_tlv_stream, invoice_tlv_stream);
+               let mut bytes = Vec::new();
+               tlv_stream.write(&mut bytes).unwrap();
+
+               let signature = merkle::sign_message(
+                       recipient_sign, INVOICE_SIGNATURE_TAG, &bytes, recipient_pubkey()
+               ).unwrap();
+               signature_tlv_stream.signature = Some(&signature);
+
+               let mut encoded_invoice = bytes;
+               signature_tlv_stream.write(&mut encoded_invoice).unwrap();
+
+               let invoice = Invoice::try_from(encoded_invoice).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
        #[test]
        fn builds_invoice_request_with_chain() {
                let mainnet = ChainHash::using_genesis_block(Network::Bitcoin);
@@ -1008,12 +1375,28 @@ mod tests {
                }
        }
 
+       #[test]
+       fn fails_responding_with_unknown_required_features() {
+               match OfferBuilder::new("foo".into(), recipient_pubkey())
+                       .amount_msats(1000)
+                       .build().unwrap()
+                       .request_invoice(vec![42; 32], payer_pubkey()).unwrap()
+                       .features_unchecked(InvoiceRequestFeatures::unknown())
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), now())
+               {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::UnknownRequiredFeatures),
+               }
+       }
+
        #[test]
        fn parses_invoice_request_with_metadata() {
                let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey())
                        .amount_msats(1000)
                        .build().unwrap()
-                       .request_invoice(vec![42; 32], payer_pubkey()).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
                        .build().unwrap()
                        .sign(payer_sign).unwrap();
 
index 9782dc7d1e84131f1d6e659dce2d28ac7e9de6f0..3b05899a8f59214872a1075179fc9428b48412fa 100644 (file)
@@ -66,7 +66,7 @@ pub(super) fn verify_signature(
        secp_ctx.verify_schnorr(signature, &digest, &pubkey)
 }
 
-fn message_digest(tag: &str, bytes: &[u8]) -> Message {
+pub(super) fn message_digest(tag: &str, bytes: &[u8]) -> Message {
        let tag = sha256::Hash::hash(tag.as_bytes());
        let merkle_root = root_hash(bytes);
        Message::from_slice(&tagged_hash(tag, merkle_root)).unwrap()
@@ -143,28 +143,38 @@ fn tagged_branch_hash_from_engine(
 
 /// [`Iterator`] over a sequence of bytes yielding [`TlvRecord`]s. The input is assumed to be a
 /// well-formed TLV stream.
-struct TlvStream<'a> {
+#[derive(Clone)]
+pub(super) struct TlvStream<'a> {
        data: io::Cursor<&'a [u8]>,
 }
 
 impl<'a> TlvStream<'a> {
-       fn new(data: &'a [u8]) -> Self {
+       pub fn new(data: &'a [u8]) -> Self {
                Self {
                        data: io::Cursor::new(data),
                }
        }
 
+       pub fn range<T>(self, types: T) -> impl core::iter::Iterator<Item = TlvRecord<'a>>
+       where
+               T: core::ops::RangeBounds<u64> + Clone,
+       {
+               let take_range = types.clone();
+               self.skip_while(move |record| !types.contains(&record.r#type))
+                       .take_while(move |record| take_range.contains(&record.r#type))
+       }
+
        fn skip_signatures(self) -> core::iter::Filter<TlvStream<'a>, fn(&TlvRecord) -> bool> {
                self.filter(|record| !SIGNATURE_TYPES.contains(&record.r#type))
        }
 }
 
 /// A slice into a [`TlvStream`] for a record.
-struct TlvRecord<'a> {
-       r#type: u64,
+pub(super) struct TlvRecord<'a> {
+       pub(super) r#type: u64,
        type_bytes: &'a [u8],
        // The entire TLV record.
-       record_bytes: &'a [u8],
+       pub(super) record_bytes: &'a [u8],
 }
 
 impl<'a> Iterator for TlvStream<'a> {
@@ -212,7 +222,7 @@ impl<'a> Writeable for WithoutSignatures<'a> {
 
 #[cfg(test)]
 mod tests {
-       use super::{TlvStream, WithoutSignatures};
+       use super::{SIGNATURE_TYPES, TlvStream, WithoutSignatures};
 
        use bitcoin::hashes::{Hash, sha256};
        use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey};
@@ -302,6 +312,38 @@ mod tests {
                );
        }
 
+       #[test]
+       fn iterates_over_tlv_stream_range() {
+               let secp_ctx = Secp256k1::new();
+               let recipient_pubkey = {
+                       let secret_key = SecretKey::from_slice(&[41; 32]).unwrap();
+                       KeyPair::from_secret_key(&secp_ctx, &secret_key).public_key()
+               };
+               let payer_keys = {
+                       let secret_key = SecretKey::from_slice(&[42; 32]).unwrap();
+                       KeyPair::from_secret_key(&secp_ctx, &secret_key)
+               };
+
+               let invoice_request = OfferBuilder::new("foo".into(), recipient_pubkey)
+                       .amount_msats(100)
+                       .build_unchecked()
+                       .request_invoice(vec![0; 8], payer_keys.public_key()).unwrap()
+                       .build_unchecked()
+                       .sign::<_, Infallible>(|digest| Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &payer_keys)))
+                       .unwrap();
+
+               let tlv_stream = TlvStream::new(&invoice_request.bytes).range(0..1)
+                       .chain(TlvStream::new(&invoice_request.bytes).range(1..80))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(80..160))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(160..240))
+                       .chain(TlvStream::new(&invoice_request.bytes).range(SIGNATURE_TYPES))
+                       .map(|r| r.record_bytes.to_vec())
+                       .flatten()
+                       .collect::<Vec<u8>>();
+
+               assert_eq!(tlv_stream, invoice_request.bytes);
+       }
+
        impl AsRef<[u8]> for InvoiceRequest {
                fn as_ref(&self) -> &[u8] {
                        &self.bytes
index 2da6fac08ff929e788af18ad1b8970ad71388730..0fb20f42d79e61b394cf46b59e6794e7b42a76fe 100644 (file)
@@ -19,3 +19,7 @@ pub mod offer;
 pub mod parse;
 mod payer;
 pub mod refund;
+#[allow(unused)]
+pub(crate) mod signer;
+#[cfg(test)]
+mod test_utils;
index 405e2e278d8073eac345a801f88906422989f173..ac16b6ef2536520929e88b6de6a14fafebf9f75d 100644 (file)
@@ -27,7 +27,7 @@
 //! use lightning::offers::parse::ParseError;
 //! use lightning::util::ser::{Readable, Writeable};
 //!
-//! # use lightning::onion_message::BlindedPath;
+//! # use lightning::blinded_path::BlindedPath;
 //! # #[cfg(feature = "std")]
 //! # use std::time::SystemTime;
 //! #
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, self};
 use core::convert::TryFrom;
 use core::num::NonZeroU64;
+use core::ops::Deref;
 use core::str::FromStr;
 use core::time::Duration;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
+use crate::blinded_path::BlindedPath;
 use crate::ln::features::OfferFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::MAX_VALUE_MSAT;
-use crate::offers::invoice_request::InvoiceRequestBuilder;
+use crate::offers::invoice_request::{DerivedPayerId, ExplicitPayerId, InvoiceRequestBuilder};
+use crate::offers::merkle::TlvStream;
 use crate::offers::parse::{Bech32Encode, ParseError, ParsedMessage, SemanticError};
-use crate::onion_message::BlindedPath;
+use crate::offers::signer::{Metadata, MetadataMaterial, self};
 use crate::util::ser::{HighZeroBytesDroppedBigSize, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
 
@@ -87,30 +92,98 @@ use crate::prelude::*;
 #[cfg(feature = "std")]
 use std::time::SystemTime;
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Offer ~~~~~~";
+
 /// Builds an [`Offer`] for the "offer to be paid" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+///
 /// [module-level documentation]: self
-pub struct OfferBuilder {
+pub struct OfferBuilder<'a, M: MetadataStrategy, T: secp256k1::Signing> {
        offer: OfferContents,
+       metadata_strategy: core::marker::PhantomData<M>,
+       secp_ctx: Option<&'a Secp256k1<T>>,
 }
 
-impl OfferBuilder {
+/// Indicates how [`Offer::metadata`] may be set.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub trait MetadataStrategy {}
+
+/// [`Offer::metadata`] may be explicitly set or left empty.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct ExplicitMetadata {}
+
+/// [`Offer::metadata`] will be derived.
+///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+pub struct DerivedMetadata {}
+
+impl MetadataStrategy for ExplicitMetadata {}
+impl MetadataStrategy for DerivedMetadata {}
+
+impl<'a> OfferBuilder<'a, ExplicitMetadata, secp256k1::SignOnly> {
        /// Creates a new builder for an offer setting the [`Offer::description`] and using the
        /// [`Offer::signing_pubkey`] for signing invoices. The associated secret key must be remembered
        /// while the offer is valid.
        ///
        /// Use a different pubkey per offer to avoid correlating offers.
        pub fn new(description: String, signing_pubkey: PublicKey) -> Self {
-               let offer = OfferContents {
-                       chains: None, metadata: None, amount: None, description,
-                       features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
-                       supported_quantity: Quantity::One, signing_pubkey,
-               };
-               OfferBuilder { offer }
+               OfferBuilder {
+                       offer: OfferContents {
+                               chains: None, metadata: None, amount: None, description,
+                               features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                               supported_quantity: Quantity::One, signing_pubkey,
+                       },
+                       metadata_strategy: core::marker::PhantomData,
+                       secp_ctx: None,
+               }
+       }
+
+       /// Sets the [`Offer::metadata`] to the given bytes.
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn metadata(mut self, metadata: Vec<u8>) -> Result<Self, SemanticError> {
+               self.offer.metadata = Some(Metadata::Bytes(metadata));
+               Ok(self)
        }
+}
 
+impl<'a, T: secp256k1::Signing> OfferBuilder<'a, DerivedMetadata, T> {
+       /// Similar to [`OfferBuilder::new`] except, if [`OfferBuilder::path`] is called, the signing
+       /// pubkey is derived from the given [`ExpandedKey`] and [`EntropySource`]. This provides
+       /// recipient privacy by using a different signing pubkey for each offer. Otherwise, the
+       /// provided `node_id` is used for the signing pubkey.
+       ///
+       /// Also, sets the metadata when [`OfferBuilder::build`] is called such that it can be used by
+       /// [`InvoiceRequest::verify`] to determine if the request was produced for the offer given an
+       /// [`ExpandedKey`].
+       ///
+       /// [`InvoiceRequest::verify`]: crate::offers::invoice_request::InvoiceRequest::verify
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn deriving_signing_pubkey<ES: Deref>(
+               description: String, node_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES,
+               secp_ctx: &'a Secp256k1<T>
+       ) -> Self where ES::Target: EntropySource {
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               OfferBuilder {
+                       offer: OfferContents {
+                               chains: None, metadata: Some(metadata), amount: None, description,
+                               features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                               supported_quantity: Quantity::One, signing_pubkey: node_id,
+                       },
+                       metadata_strategy: core::marker::PhantomData,
+                       secp_ctx: Some(secp_ctx),
+               }
+       }
+}
+
+impl<'a, M: MetadataStrategy, T: secp256k1::Signing> OfferBuilder<'a, M, T> {
        /// Adds the chain hash of the given [`Network`] to [`Offer::chains`]. If not called,
        /// the chain hash of [`Network::Bitcoin`] is assumed to be the only one supported.
        ///
@@ -127,14 +200,6 @@ impl OfferBuilder {
                self
        }
 
-       /// Sets the [`Offer::metadata`].
-       ///
-       /// Successive calls to this method will override the previous setting.
-       pub fn metadata(mut self, metadata: Vec<u8>) -> Self {
-               self.offer.metadata = Some(metadata);
-               self
-       }
-
        /// Sets the [`Offer::amount`] as an [`Amount::Bitcoin`].
        ///
        /// Successive calls to this method will override the previous setting.
@@ -204,28 +269,50 @@ impl OfferBuilder {
                        }
                }
 
+               Ok(self.build_without_checks())
+       }
+
+       fn build_without_checks(mut self) -> Offer {
+               // Create the metadata for stateless verification of an InvoiceRequest.
+               if let Some(mut metadata) = self.offer.metadata.take() {
+                       if metadata.has_derivation_material() {
+                               if self.offer.paths.is_none() {
+                                       metadata = metadata.without_keys();
+                               }
+
+                               let mut tlv_stream = self.offer.as_tlv_stream();
+                               debug_assert_eq!(tlv_stream.metadata, None);
+                               tlv_stream.metadata = None;
+                               if metadata.derives_keys() {
+                                       tlv_stream.node_id = None;
+                               }
+
+                               let (derived_metadata, keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                               metadata = derived_metadata;
+                               if let Some(keys) = keys {
+                                       self.offer.signing_pubkey = keys.public_key();
+                               }
+                       }
+
+                       self.offer.metadata = Some(metadata);
+               }
+
                let mut bytes = Vec::new();
                self.offer.write(&mut bytes).unwrap();
 
-               Ok(Offer {
-                       bytes,
-                       contents: self.offer,
-               })
+               Offer { bytes, contents: self.offer }
        }
 }
 
 #[cfg(test)]
-impl OfferBuilder {
+impl<'a, M: MetadataStrategy, T: secp256k1::Signing> OfferBuilder<'a, M, T> {
        fn features_unchecked(mut self, features: OfferFeatures) -> Self {
                self.offer.features = features;
                self
        }
 
        pub(super) fn build_unchecked(self) -> Offer {
-               let mut bytes = Vec::new();
-               self.offer.write(&mut bytes).unwrap();
-
-               Offer { bytes, contents: self.offer }
+               self.build_without_checks()
        }
 }
 
@@ -242,7 +329,8 @@ impl OfferBuilder {
 ///
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Offer {
        // The serialized offer. Needed when creating an `InvoiceRequest` if the offer contains unknown
        // fields.
@@ -254,10 +342,11 @@ pub struct Offer {
 ///
 /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct OfferContents {
        chains: Option<Vec<ChainHash>>,
-       metadata: Option<Vec<u8>>,
+       metadata: Option<Metadata>,
        amount: Option<Amount>,
        description: String,
        features: OfferFeatures,
@@ -292,7 +381,7 @@ impl Offer {
        /// Opaque bytes set by the originator. Useful for authentication and validating fields since it
        /// is reflected in `invoice_request` messages along with all the other fields from the `offer`.
        pub fn metadata(&self) -> Option<&Vec<u8>> {
-               self.contents.metadata.as_ref()
+               self.contents.metadata()
        }
 
        /// The minimum amount required for a successful payment of a single item.
@@ -303,7 +392,7 @@ impl Offer {
        /// A complete description of the purpose of the payment. Intended to be displayed to the user
        /// but with the caveat that it has not been verified in any way.
        pub fn description(&self) -> PrintableString {
-               PrintableString(&self.contents.description)
+               self.contents.description()
        }
 
        /// Features pertaining to the offer.
@@ -358,8 +447,57 @@ impl Offer {
                self.contents.signing_pubkey()
        }
 
-       /// Creates an [`InvoiceRequest`] for the offer with the given `metadata` and `payer_id`, which
-       /// will be reflected in the `Invoice` response.
+       /// Similar to [`Offer::request_invoice`] except it:
+       /// - derives the [`InvoiceRequest::payer_id`] such that a different key can be used for each
+       ///   request, and
+       /// - sets the [`InvoiceRequest::metadata`] when [`InvoiceRequestBuilder::build`] is called such
+       ///   that it can be used by [`Invoice::verify`] to determine if the invoice was requested using
+       ///   a base [`ExpandedKey`] from which the payer id was derived.
+       ///
+       /// Useful to protect the sender's privacy.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
+       /// [`InvoiceRequest::metadata`]: crate::offers::invoice_request::InvoiceRequest::metadata
+       /// [`Invoice::verify`]: crate::offers::invoice::Invoice::verify
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn request_invoice_deriving_payer_id<'a, 'b, ES: Deref, T: secp256k1::Signing>(
+               &'a self, expanded_key: &ExpandedKey, entropy_source: ES, secp_ctx: &'b Secp256k1<T>
+       ) -> Result<InvoiceRequestBuilder<'a, 'b, DerivedPayerId, T>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               Ok(InvoiceRequestBuilder::deriving_payer_id(self, expanded_key, entropy_source, secp_ctx))
+       }
+
+       /// Similar to [`Offer::request_invoice_deriving_payer_id`] except uses `payer_id` for the
+       /// [`InvoiceRequest::payer_id`] instead of deriving a different key for each request.
+       ///
+       /// Useful for recurring payments using the same `payer_id` with different invoices.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
+       pub fn request_invoice_deriving_metadata<ES: Deref>(
+               &self, payer_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceRequestBuilder<ExplicitPayerId, secp256k1::SignOnly>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               Ok(InvoiceRequestBuilder::deriving_metadata(self, payer_id, expanded_key, entropy_source))
+       }
+
+       /// Creates an [`InvoiceRequestBuilder`] for the offer with the given `metadata` and `payer_id`,
+       /// which will be reflected in the `Invoice` response.
        ///
        /// The `metadata` is useful for including information about the derivation of `payer_id` such
        /// that invoice response handling can be stateless. Also serves as payer-provided entropy while
@@ -370,10 +508,12 @@ impl Offer {
        ///
        /// Errors if the offer contains unknown required features.
        ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
        /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
        pub fn request_invoice(
                &self, metadata: Vec<u8>, payer_id: PublicKey
-       ) -> Result<InvoiceRequestBuilder, SemanticError> {
+       ) -> Result<InvoiceRequestBuilder<ExplicitPayerId, secp256k1::SignOnly>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -406,6 +546,14 @@ impl OfferContents {
                self.chains().contains(&chain)
        }
 
+       pub fn metadata(&self) -> Option<&Vec<u8>> {
+               self.metadata.as_ref().and_then(|metadata| metadata.as_bytes())
+       }
+
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.description)
+       }
+
        #[cfg(feature = "std")]
        pub(super) fn is_expired(&self) -> bool {
                match self.absolute_expiry {
@@ -483,6 +631,27 @@ impl OfferContents {
                self.signing_pubkey
        }
 
+       /// Verifies that the offer metadata was produced from the offer in the TLV stream.
+       pub(super) fn verify<T: secp256k1::Signing>(
+               &self, bytes: &[u8], key: &ExpandedKey, secp_ctx: &Secp256k1<T>
+       ) -> Result<Option<KeyPair>, ()> {
+               match self.metadata() {
+                       Some(metadata) => {
+                               let tlv_stream = TlvStream::new(bytes).range(OFFER_TYPES).filter(|record| {
+                                       match record.r#type {
+                                               OFFER_METADATA_TYPE => false,
+                                               OFFER_NODE_ID_TYPE => !self.metadata.as_ref().unwrap().derives_keys(),
+                                               _ => true,
+                                       }
+                               });
+                               signer::verify_metadata(
+                                       metadata, key, IV_BYTES, self.signing_pubkey(), tlv_stream, secp_ctx
+                               )
+                       },
+                       None => Err(()),
+               }
+       }
+
        pub(super) fn as_tlv_stream(&self) -> OfferTlvStreamRef {
                let (currency, amount) = match &self.amount {
                        None => (None, None),
@@ -498,7 +667,7 @@ impl OfferContents {
 
                OfferTlvStreamRef {
                        chains: self.chains.as_ref(),
-                       metadata: self.metadata.as_ref(),
+                       metadata: self.metadata(),
                        currency,
                        amount,
                        description: Some(&self.description),
@@ -570,9 +739,18 @@ impl Quantity {
        }
 }
 
-tlv_stream!(OfferTlvStream, OfferTlvStreamRef, 1..80, {
+/// Valid type range for offer TLV records.
+pub(super) const OFFER_TYPES: core::ops::Range<u64> = 1..80;
+
+/// TLV record type for [`Offer::metadata`].
+const OFFER_METADATA_TYPE: u64 = 4;
+
+/// TLV record type for [`Offer::signing_pubkey`].
+const OFFER_NODE_ID_TYPE: u64 = 22;
+
+tlv_stream!(OfferTlvStream, OfferTlvStreamRef, OFFER_TYPES, {
        (2, chains: (Vec<ChainHash>, WithoutLength)),
-       (4, metadata: (Vec<u8>, WithoutLength)),
+       (OFFER_METADATA_TYPE, metadata: (Vec<u8>, WithoutLength)),
        (6, currency: CurrencyCode),
        (8, amount: (u64, HighZeroBytesDroppedBigSize)),
        (10, description: (String, WithoutLength)),
@@ -581,7 +759,7 @@ tlv_stream!(OfferTlvStream, OfferTlvStreamRef, 1..80, {
        (16, paths: (Vec<BlindedPath>, WithoutLength)),
        (18, issuer: (String, WithoutLength)),
        (20, quantity_max: (u64, HighZeroBytesDroppedBigSize)),
-       (22, node_id: PublicKey),
+       (OFFER_NODE_ID_TYPE, node_id: PublicKey),
 });
 
 impl Bech32Encode for Offer {
@@ -616,6 +794,8 @@ impl TryFrom<OfferTlvStream> for OfferContents {
                        issuer, quantity_max, node_id,
                } = tlv_stream;
 
+               let metadata = metadata.map(|metadata| Metadata::Bytes(metadata));
+
                let amount = match (currency, amount) {
                        (None, None) => None,
                        (None, Some(amount_msats)) if amount_msats > MAX_VALUE_MSAT => {
@@ -666,26 +846,20 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
+       use bitcoin::secp256k1::Secp256k1;
        use core::convert::TryFrom;
        use core::num::NonZeroU64;
        use core::time::Duration;
+       use crate::blinded_path::{BlindedHop, BlindedPath};
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::OfferFeatures;
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
        use crate::offers::parse::{ParseError, SemanticError};
-       use crate::onion_message::{BlindedHop, BlindedPath};
+       use crate::offers::test_utils::*;
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
-
        #[test]
        fn builds_offer_with_defaults() {
                let offer = OfferBuilder::new("foo".into(), pubkey(42)).build().unwrap();
@@ -774,21 +948,125 @@ mod tests {
        #[test]
        fn builds_offer_with_metadata() {
                let offer = OfferBuilder::new("foo".into(), pubkey(42))
-                       .metadata(vec![42; 32])
+                       .metadata(vec![42; 32]).unwrap()
                        .build()
                        .unwrap();
                assert_eq!(offer.metadata(), Some(&vec![42; 32]));
                assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![42; 32]));
 
                let offer = OfferBuilder::new("foo".into(), pubkey(42))
-                       .metadata(vec![42; 32])
-                       .metadata(vec![43; 32])
+                       .metadata(vec![42; 32]).unwrap()
+                       .metadata(vec![43; 32]).unwrap()
                        .build()
                        .unwrap();
                assert_eq!(offer.metadata(), Some(&vec![43; 32]));
                assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![43; 32]));
        }
 
+       #[test]
+       fn builds_offer_with_metadata_derived() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .build().unwrap();
+               assert_eq!(offer.signing_pubkey(), node_id);
+
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_ok());
+
+               // Fails verification with altered offer field
+               let mut tlv_stream = offer.as_tlv_stream();
+               tlv_stream.amount = Some(100);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+
+               // Fails verification with altered metadata
+               let mut tlv_stream = offer.as_tlv_stream();
+               let metadata = tlv_stream.metadata.unwrap().iter().copied().rev().collect();
+               tlv_stream.metadata = Some(&metadata);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+       }
+
+       #[test]
+       fn builds_offer_with_derived_signing_pubkey() {
+               let desc = "foo".to_string();
+               let node_id = recipient_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(42), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let offer = OfferBuilder
+                       ::deriving_signing_pubkey(desc, node_id, &expanded_key, &entropy, &secp_ctx)
+                       .amount_msats(1000)
+                       .path(blinded_path)
+                       .build().unwrap();
+               assert_ne!(offer.signing_pubkey(), node_id);
+
+               let invoice_request = offer.request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_ok());
+
+               // Fails verification with altered offer field
+               let mut tlv_stream = offer.as_tlv_stream();
+               tlv_stream.amount = Some(100);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+
+               // Fails verification with altered signing pubkey
+               let mut tlv_stream = offer.as_tlv_stream();
+               let signing_pubkey = pubkey(1);
+               tlv_stream.node_id = Some(&signing_pubkey);
+
+               let mut encoded_offer = Vec::new();
+               tlv_stream.write(&mut encoded_offer).unwrap();
+
+               let invoice_request = Offer::try_from(encoded_offer).unwrap()
+                       .request_invoice(vec![1; 32], payer_pubkey()).unwrap()
+                       .build().unwrap()
+                       .sign(payer_sign).unwrap();
+               assert!(invoice_request.verify(&expanded_key, &secp_ctx).is_err());
+       }
+
        #[test]
        fn builds_offer_with_amount() {
                let bitcoin_amount = Amount::Bitcoin { amount_msats: 1000 };
index 6afd4d68fef385bb5b6b77b8375891e53878f6e7..42ed2e002d1f29b60169c86954f39cece3101647 100644 (file)
@@ -116,6 +116,8 @@ impl<T: SeekReadable> TryFrom<Vec<u8>> for ParsedMessage<T> {
 }
 
 /// Error when parsing a bech32 encoded message using [`str::parse`].
+///
+/// This is not exported to bindings users as its name conflicts with the BOLT 11 ParseError type.
 #[derive(Debug, PartialEq)]
 pub enum ParseError {
        /// The bech32 encoding does not conform to the BOLT 12 requirements for continuing messages
@@ -135,6 +137,8 @@ pub enum ParseError {
 }
 
 /// Error when interpreting a TLV stream as a specific type.
+///
+/// This is not exported to bindings users as its name conflicts with the BOLT 11 SemanticError type.
 #[derive(Debug, PartialEq)]
 pub enum SemanticError {
        /// The current [`std::time::SystemTime`] is past the offer or invoice's expiration.
@@ -171,6 +175,8 @@ pub enum SemanticError {
        InvalidQuantity,
        /// A quantity or quantity bounds was provided but was not expected.
        UnexpectedQuantity,
+       /// Metadata could not be used to verify the offers message.
+       InvalidMetadata,
        /// Metadata was provided but was not expected.
        UnexpectedMetadata,
        /// Payer metadata was expected but was missing.
index 7e1da769edab6cec527c4de425bb7711fc50fa1f..bfc02b5dbcb9804f1fd08bdf689f541a12579ed9 100644 (file)
@@ -9,6 +9,7 @@
 
 //! Data structures and encoding for `invoice_request_metadata` records.
 
+use crate::offers::signer::Metadata;
 use crate::util::ser::WithoutLength;
 
 use crate::prelude::*;
@@ -17,9 +18,16 @@ use crate::prelude::*;
 /// [`InvoiceRequest::payer_id`].
 ///
 /// [`InvoiceRequest::payer_id`]: crate::offers::invoice_request::InvoiceRequest::payer_id
-#[derive(Clone, Debug, PartialEq)]
-pub(super) struct PayerContents(pub Vec<u8>);
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
+pub(super) struct PayerContents(pub Metadata);
+
+/// TLV record type for [`InvoiceRequest::metadata`] and [`Refund::metadata`].
+///
+/// [`InvoiceRequest::metadata`]: crate::offers::invoice_request::InvoiceRequest::metadata
+/// [`Refund::metadata`]: crate::offers::refund::Refund::metadata
+pub(super) const PAYER_METADATA_TYPE: u64 = 0;
 
 tlv_stream!(PayerTlvStream, PayerTlvStreamRef, 0..1, {
-       (0, metadata: (Vec<u8>, WithoutLength)),
+       (PAYER_METADATA_TYPE, metadata: (Vec<u8>, WithoutLength)),
 });
index cc0388c0241b5f77a125ab0dabe4da8427504e8f..6cbdd2da2bc091e70c731942970fabec55ba2026 100644 (file)
@@ -32,7 +32,7 @@
 //! use lightning::offers::refund::{Refund, RefundBuilder};
 //! use lightning::util::ser::{Readable, Writeable};
 //!
-//! # use lightning::onion_message::BlindedPath;
+//! # use lightning::blinded_path::BlindedPath;
 //! # #[cfg(feature = "std")]
 //! # use std::time::SystemTime;
 //! #
 
 use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::network::constants::Network;
-use bitcoin::secp256k1::PublicKey;
+use bitcoin::secp256k1::{PublicKey, Secp256k1, self};
 use core::convert::TryFrom;
+use core::ops::Deref;
 use core::str::FromStr;
 use core::time::Duration;
+use crate::chain::keysinterface::EntropySource;
 use crate::io;
+use crate::blinded_path::BlindedPath;
 use crate::ln::PaymentHash;
 use crate::ln::features::InvoiceRequestFeatures;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
 use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
-use crate::offers::invoice::{BlindedPayInfo, InvoiceBuilder};
+use crate::offers::invoice::{BlindedPayInfo, DerivedSigningPubkey, ExplicitSigningPubkey, InvoiceBuilder};
 use crate::offers::invoice_request::{InvoiceRequestTlvStream, InvoiceRequestTlvStreamRef};
 use crate::offers::offer::{OfferTlvStream, OfferTlvStreamRef};
 use crate::offers::parse::{Bech32Encode, ParseError, ParsedMessage, SemanticError};
 use crate::offers::payer::{PayerContents, PayerTlvStream, PayerTlvStreamRef};
-use crate::onion_message::BlindedPath;
+use crate::offers::signer::{Metadata, MetadataMaterial, self};
 use crate::util::ser::{SeekReadable, WithoutLength, Writeable, Writer};
 use crate::util::string::PrintableString;
 
@@ -95,16 +99,21 @@ use crate::prelude::*;
 #[cfg(feature = "std")]
 use std::time::SystemTime;
 
+pub(super) const IV_BYTES: &[u8; IV_LEN] = b"LDK Refund ~~~~~";
+
 /// Builds a [`Refund`] for the "offer for money" flow.
 ///
 /// See [module-level documentation] for usage.
 ///
+/// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+///
 /// [module-level documentation]: self
-pub struct RefundBuilder {
+pub struct RefundBuilder<'a, T: secp256k1::Signing> {
        refund: RefundContents,
+       secp_ctx: Option<&'a Secp256k1<T>>,
 }
 
-impl RefundBuilder {
+impl<'a> RefundBuilder<'a, secp256k1::SignOnly> {
        /// Creates a new builder for a refund using the [`Refund::payer_id`] for the public node id to
        /// send to if no [`Refund::paths`] are set. Otherwise, it may be a transient pubkey.
        ///
@@ -117,13 +126,48 @@ impl RefundBuilder {
                        return Err(SemanticError::InvalidAmount);
                }
 
-               let refund = RefundContents {
-                       payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
-                       paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
-                       quantity: None, payer_id, payer_note: None,
-               };
+               let metadata = Metadata::Bytes(metadata);
+               Ok(Self {
+                       refund: RefundContents {
+                               payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
+                               paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
+                               quantity: None, payer_id, payer_note: None,
+                       },
+                       secp_ctx: None,
+               })
+       }
+}
 
-               Ok(RefundBuilder { refund })
+impl<'a, T: secp256k1::Signing> RefundBuilder<'a, T> {
+       /// Similar to [`RefundBuilder::new`] except, if [`RefundBuilder::path`] is called, the payer id
+       /// is derived from the given [`ExpandedKey`] and nonce. This provides sender privacy by using a
+       /// different payer id for each refund, assuming a different nonce is used.  Otherwise, the
+       /// provided `node_id` is used for the payer id.
+       ///
+       /// Also, sets the metadata when [`RefundBuilder::build`] is called such that it can be used to
+       /// verify that an [`InvoiceRequest`] was produced for the refund given an [`ExpandedKey`].
+       ///
+       /// [`InvoiceRequest`]: crate::offers::invoice_request::InvoiceRequest
+       /// [`ExpandedKey`]: crate::ln::inbound_payment::ExpandedKey
+       pub fn deriving_payer_id<ES: Deref>(
+               description: String, node_id: PublicKey, expanded_key: &ExpandedKey, entropy_source: ES,
+               secp_ctx: &'a Secp256k1<T>, amount_msats: u64
+       ) -> Result<Self, SemanticError> where ES::Target: EntropySource {
+               if amount_msats > MAX_VALUE_MSAT {
+                       return Err(SemanticError::InvalidAmount);
+               }
+
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let derivation_material = MetadataMaterial::new(nonce, expanded_key, IV_BYTES);
+               let metadata = Metadata::DerivedSigningPubkey(derivation_material);
+               Ok(Self {
+                       refund: RefundContents {
+                               payer: PayerContents(metadata), description, absolute_expiry: None, issuer: None,
+                               paths: None, chain: None, amount_msats, features: InvoiceRequestFeatures::empty(),
+                               quantity: None, payer_id: node_id, payer_note: None,
+                       },
+                       secp_ctx: Some(secp_ctx),
+               })
        }
 
        /// Sets the [`Refund::absolute_expiry`] as seconds since the Unix epoch. Any expiry that has
@@ -190,18 +234,38 @@ impl RefundBuilder {
                        self.refund.chain = None;
                }
 
+               // Create the metadata for stateless verification of an Invoice.
+               if self.refund.payer.0.has_derivation_material() {
+                       let mut metadata = core::mem::take(&mut self.refund.payer.0);
+
+                       if self.refund.paths.is_none() {
+                               metadata = metadata.without_keys();
+                       }
+
+                       let mut tlv_stream = self.refund.as_tlv_stream();
+                       tlv_stream.0.metadata = None;
+                       if metadata.derives_keys() {
+                               tlv_stream.2.payer_id = None;
+                       }
+
+                       let (derived_metadata, keys) = metadata.derive_from(tlv_stream, self.secp_ctx);
+                       metadata = derived_metadata;
+                       if let Some(keys) = keys {
+                               self.refund.payer_id = keys.public_key();
+                       }
+
+                       self.refund.payer.0 = metadata;
+               }
+
                let mut bytes = Vec::new();
                self.refund.write(&mut bytes).unwrap();
 
-               Ok(Refund {
-                       bytes,
-                       contents: self.refund,
-               })
+               Ok(Refund { bytes, contents: self.refund })
        }
 }
 
 #[cfg(test)]
-impl RefundBuilder {
+impl<'a, T: secp256k1::Signing> RefundBuilder<'a, T> {
        fn features_unchecked(mut self, features: InvoiceRequestFeatures) -> Self {
                self.refund.features = features;
                self
@@ -216,7 +280,8 @@ impl RefundBuilder {
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
 /// [`Offer`]: crate::offers::offer::Offer
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub struct Refund {
        pub(super) bytes: Vec<u8>,
        pub(super) contents: RefundContents,
@@ -225,7 +290,8 @@ pub struct Refund {
 /// The contents of a [`Refund`], which may be shared with an [`Invoice`].
 ///
 /// [`Invoice`]: crate::offers::invoice::Invoice
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug)]
+#[cfg_attr(test, derive(PartialEq))]
 pub(super) struct RefundContents {
        payer: PayerContents,
        // offer fields
@@ -246,7 +312,7 @@ impl Refund {
        /// A complete description of the purpose of the refund. Intended to be displayed to the user
        /// but with the caveat that it has not been verified in any way.
        pub fn description(&self) -> PrintableString {
-               PrintableString(&self.contents.description)
+               self.contents.description()
        }
 
        /// Duration since the Unix epoch when an invoice should no longer be sent.
@@ -279,7 +345,7 @@ impl Refund {
        ///
        /// [`payer_id`]: Self::payer_id
        pub fn metadata(&self) -> &[u8] {
-               &self.contents.payer.0
+               self.contents.metadata()
        }
 
        /// A chain that the refund is valid for.
@@ -317,19 +383,20 @@ impl Refund {
                self.contents.payer_note.as_ref().map(|payer_note| PrintableString(payer_note.as_str()))
        }
 
-       /// Creates an [`Invoice`] for the refund with the given required fields and using the
+       /// Creates an [`InvoiceBuilder`] for the refund with the given required fields and using the
        /// [`Duration`] since [`std::time::SystemTime::UNIX_EPOCH`] as the creation time.
        ///
        /// See [`Refund::respond_with_no_std`] for further details where the aforementioned creation
        /// time is used for the `created_at` parameter.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
        /// [`Duration`]: core::time::Duration
        #[cfg(feature = "std")]
        pub fn respond_with(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                signing_pubkey: PublicKey,
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                let created_at = std::time::SystemTime::now()
                        .duration_since(std::time::SystemTime::UNIX_EPOCH)
                        .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
@@ -337,7 +404,7 @@ impl Refund {
                self.respond_with_no_std(payment_paths, payment_hash, signing_pubkey, created_at)
        }
 
-       /// Creates an [`Invoice`] for the refund with the given required fields.
+       /// Creates an [`InvoiceBuilder`] for the refund with the given required fields.
        ///
        /// Unless [`InvoiceBuilder::relative_expiry`] is set, the invoice will expire two hours after
        /// `created_at`, which is used to set [`Invoice::created_at`]. Useful for `no-std` builds where
@@ -356,12 +423,13 @@ impl Refund {
        ///
        /// Errors if the request contains unknown required features.
        ///
-       /// [`Invoice`]: crate::offers::invoice::Invoice
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
        /// [`Invoice::created_at`]: crate::offers::invoice::Invoice::created_at
        pub fn respond_with_no_std(
                &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
                signing_pubkey: PublicKey, created_at: Duration
-       ) -> Result<InvoiceBuilder, SemanticError> {
+       ) -> Result<InvoiceBuilder<ExplicitSigningPubkey>, SemanticError> {
                if self.features().requires_unknown_bits() {
                        return Err(SemanticError::UnknownRequiredFeatures);
                }
@@ -369,6 +437,55 @@ impl Refund {
                InvoiceBuilder::for_refund(self, payment_paths, created_at, payment_hash, signing_pubkey)
        }
 
+       /// Creates an [`InvoiceBuilder`] for the refund using the given required fields and that uses
+       /// derived signing keys to sign the [`Invoice`].
+       ///
+       /// See [`Refund::respond_with`] for further details.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       #[cfg(feature = "std")]
+       pub fn respond_using_derived_keys<ES: Deref>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               let created_at = std::time::SystemTime::now()
+                       .duration_since(std::time::SystemTime::UNIX_EPOCH)
+                       .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH");
+
+               self.respond_using_derived_keys_no_std(
+                       payment_paths, payment_hash, created_at, expanded_key, entropy_source
+               )
+       }
+
+       /// Creates an [`InvoiceBuilder`] for the refund using the given required fields and that uses
+       /// derived signing keys to sign the [`Invoice`].
+       ///
+       /// See [`Refund::respond_with_no_std`] for further details.
+       ///
+       /// This is not exported to bindings users as builder patterns don't map outside of move semantics.
+       ///
+       /// [`Invoice`]: crate::offers::invoice::Invoice
+       pub fn respond_using_derived_keys_no_std<ES: Deref>(
+               &self, payment_paths: Vec<(BlindedPath, BlindedPayInfo)>, payment_hash: PaymentHash,
+               created_at: core::time::Duration, expanded_key: &ExpandedKey, entropy_source: ES
+       ) -> Result<InvoiceBuilder<DerivedSigningPubkey>, SemanticError>
+       where
+               ES::Target: EntropySource,
+       {
+               if self.features().requires_unknown_bits() {
+                       return Err(SemanticError::UnknownRequiredFeatures);
+               }
+
+               let nonce = Nonce::from_entropy_source(entropy_source);
+               let keys = signer::derive_keys(nonce, expanded_key);
+               InvoiceBuilder::for_refund_using_keys(self, payment_paths, created_at, payment_hash, keys)
+       }
+
        #[cfg(test)]
        fn as_tlv_stream(&self) -> RefundTlvStreamRef {
                self.contents.as_tlv_stream()
@@ -382,6 +499,10 @@ impl AsRef<[u8]> for Refund {
 }
 
 impl RefundContents {
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.description)
+       }
+
        #[cfg(feature = "std")]
        pub(super) fn is_expired(&self) -> bool {
                match self.absolute_expiry {
@@ -393,6 +514,10 @@ impl RefundContents {
                }
        }
 
+       pub(super) fn metadata(&self) -> &[u8] {
+               self.payer.0.as_bytes().map(|bytes| bytes.as_slice()).unwrap_or(&[])
+       }
+
        pub(super) fn chain(&self) -> ChainHash {
                self.chain.unwrap_or_else(|| self.implied_chain())
        }
@@ -401,9 +526,17 @@ impl RefundContents {
                ChainHash::using_genesis_block(Network::Bitcoin)
        }
 
+       pub(super) fn derives_keys(&self) -> bool {
+               self.payer.0.derives_keys()
+       }
+
+       pub(super) fn payer_id(&self) -> PublicKey {
+               self.payer_id
+       }
+
        pub(super) fn as_tlv_stream(&self) -> RefundTlvStreamRef {
                let payer = PayerTlvStreamRef {
-                       metadata: Some(&self.payer.0),
+                       metadata: self.payer.0.as_bytes(),
                };
 
                let offer = OfferTlvStreamRef {
@@ -507,7 +640,7 @@ impl TryFrom<RefundTlvStream> for RefundContents {
 
                let payer = match payer_metadata {
                        None => return Err(SemanticError::MissingPayerMetadata),
-                       Some(metadata) => PayerContents(metadata),
+                       Some(metadata) => PayerContents(Metadata::Bytes(metadata)),
                };
 
                if metadata.is_some() {
@@ -575,33 +708,22 @@ mod tests {
 
        use bitcoin::blockdata::constants::ChainHash;
        use bitcoin::network::constants::Network;
-       use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey};
+       use bitcoin::secp256k1::{KeyPair, Secp256k1, SecretKey};
        use core::convert::TryFrom;
        use core::time::Duration;
+       use crate::blinded_path::{BlindedHop, BlindedPath};
+       use crate::chain::keysinterface::KeyMaterial;
        use crate::ln::features::{InvoiceRequestFeatures, OfferFeatures};
+       use crate::ln::inbound_payment::ExpandedKey;
        use crate::ln::msgs::{DecodeError, MAX_VALUE_MSAT};
        use crate::offers::invoice_request::InvoiceRequestTlvStreamRef;
        use crate::offers::offer::OfferTlvStreamRef;
        use crate::offers::parse::{ParseError, SemanticError};
        use crate::offers::payer::PayerTlvStreamRef;
-       use crate::onion_message::{BlindedHop, BlindedPath};
+       use crate::offers::test_utils::*;
        use crate::util::ser::{BigSize, Writeable};
        use crate::util::string::PrintableString;
 
-       fn payer_pubkey() -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap()).public_key()
-       }
-
-       fn pubkey(byte: u8) -> PublicKey {
-               let secp_ctx = Secp256k1::new();
-               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
-       }
-
-       fn privkey(byte: u8) -> SecretKey {
-               SecretKey::from_slice(&[byte; 32]).unwrap()
-       }
-
        trait ToBytes {
                fn to_bytes(&self) -> Vec<u8>;
        }
@@ -677,6 +799,118 @@ mod tests {
                }
        }
 
+       #[test]
+       fn builds_refund_with_metadata_derived() {
+               let desc = "foo".to_string();
+               let node_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let refund = RefundBuilder
+                       ::deriving_payer_id(desc, node_id, &expanded_key, &entropy, &secp_ctx, 1000)
+                       .unwrap()
+                       .build().unwrap();
+               assert_eq!(refund.payer_id(), node_id);
+
+               // Fails verification with altered fields
+               let invoice = refund
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               let mut tlv_stream = refund.as_tlv_stream();
+               tlv_stream.2.amount = Some(2000);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered metadata
+               let mut tlv_stream = refund.as_tlv_stream();
+               let metadata = tlv_stream.0.metadata.unwrap().iter().copied().rev().collect();
+               tlv_stream.0.metadata = Some(&metadata);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
+       #[test]
+       fn builds_refund_with_derived_payer_id() {
+               let desc = "foo".to_string();
+               let node_id = payer_pubkey();
+               let expanded_key = ExpandedKey::new(&KeyMaterial([42; 32]));
+               let entropy = FixedEntropy {};
+               let secp_ctx = Secp256k1::new();
+
+               let blinded_path = BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: node_id, encrypted_payload: vec![0; 44] },
+                       ],
+               };
+
+               let refund = RefundBuilder
+                       ::deriving_payer_id(desc, node_id, &expanded_key, &entropy, &secp_ctx, 1000)
+                       .unwrap()
+                       .path(blinded_path)
+                       .build().unwrap();
+               assert_ne!(refund.payer_id(), node_id);
+
+               let invoice = refund
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered fields
+               let mut tlv_stream = refund.as_tlv_stream();
+               tlv_stream.2.amount = Some(2000);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+
+               // Fails verification with altered payer_id
+               let mut tlv_stream = refund.as_tlv_stream();
+               let payer_id = pubkey(1);
+               tlv_stream.2.payer_id = Some(&payer_id);
+
+               let mut encoded_refund = Vec::new();
+               tlv_stream.write(&mut encoded_refund).unwrap();
+
+               let invoice = Refund::try_from(encoded_refund).unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+                       .unwrap()
+                       .build().unwrap()
+                       .sign(recipient_sign).unwrap();
+               assert!(!invoice.verify(&expanded_key, &secp_ctx));
+       }
+
        #[test]
        fn builds_refund_with_absolute_expiry() {
                let future_expiry = Duration::from_secs(u64::max_value());
@@ -822,6 +1056,18 @@ mod tests {
                assert_eq!(tlv_stream.payer_note, Some(&String::from("baz")));
        }
 
+       #[test]
+       fn fails_responding_with_unknown_required_features() {
+               match RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
+                       .features_unchecked(InvoiceRequestFeatures::unknown())
+                       .build().unwrap()
+                       .respond_with_no_std(payment_paths(), payment_hash(), recipient_pubkey(), now())
+               {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, SemanticError::UnknownRequiredFeatures),
+               }
+       }
+
        #[test]
        fn parses_refund_with_metadata() {
                let refund = RefundBuilder::new("foo".into(), vec![1; 32], payer_pubkey(), 1000).unwrap()
diff --git a/lightning/src/offers/signer.rs b/lightning/src/offers/signer.rs
new file mode 100644 (file)
index 0000000..8d5f98e
--- /dev/null
@@ -0,0 +1,231 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for signing offer messages and verifying metadata.
+
+use bitcoin::hashes::{Hash, HashEngine};
+use bitcoin::hashes::cmp::fixed_time_eq;
+use bitcoin::hashes::hmac::{Hmac, HmacEngine};
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey, self};
+use core::convert::TryFrom;
+use core::fmt;
+use crate::ln::inbound_payment::{ExpandedKey, IV_LEN, Nonce};
+use crate::offers::merkle::TlvRecord;
+use crate::util::ser::Writeable;
+
+use crate::prelude::*;
+
+const DERIVED_METADATA_HMAC_INPUT: &[u8; 16] = &[1; 16];
+const DERIVED_METADATA_AND_KEYS_HMAC_INPUT: &[u8; 16] = &[2; 16];
+
+/// Message metadata which possibly is derived from [`MetadataMaterial`] such that it can be
+/// verified.
+#[derive(Clone)]
+pub(super) enum Metadata {
+       /// Metadata as parsed, supplied by the user, or derived from the message contents.
+       Bytes(Vec<u8>),
+
+       /// Metadata to be derived from message contents and given material.
+       Derived(MetadataMaterial),
+
+       /// Metadata and signing pubkey to be derived from message contents and given material.
+       DerivedSigningPubkey(MetadataMaterial),
+}
+
+impl Metadata {
+       pub fn as_bytes(&self) -> Option<&Vec<u8>> {
+               match self {
+                       Metadata::Bytes(bytes) => Some(bytes),
+                       Metadata::Derived(_) => None,
+                       Metadata::DerivedSigningPubkey(_) => None,
+               }
+       }
+
+       pub fn has_derivation_material(&self) -> bool {
+               match self {
+                       Metadata::Bytes(_) => false,
+                       Metadata::Derived(_) => true,
+                       Metadata::DerivedSigningPubkey(_) => true,
+               }
+       }
+
+       pub fn derives_keys(&self) -> bool {
+               match self {
+                       // Infer whether Metadata::derived_from was called on Metadata::DerivedSigningPubkey to
+                       // produce Metadata::Bytes. This is merely to determine which fields should be included
+                       // when verifying a message. It doesn't necessarily indicate that keys were in fact
+                       // derived, as wouldn't be the case if a Metadata::Bytes with length Nonce::LENGTH had
+                       // been set explicitly.
+                       Metadata::Bytes(bytes) => bytes.len() == Nonce::LENGTH,
+                       Metadata::Derived(_) => false,
+                       Metadata::DerivedSigningPubkey(_) => true,
+               }
+       }
+
+       pub fn without_keys(self) -> Self {
+               match self {
+                       Metadata::Bytes(_) => self,
+                       Metadata::Derived(_) => self,
+                       Metadata::DerivedSigningPubkey(material) => Metadata::Derived(material),
+               }
+       }
+
+       pub fn derive_from<W: Writeable, T: secp256k1::Signing>(
+               self, tlv_stream: W, secp_ctx: Option<&Secp256k1<T>>
+       ) -> (Self, Option<KeyPair>) {
+               match self {
+                       Metadata::Bytes(_) => (self, None),
+                       Metadata::Derived(mut metadata_material) => {
+                               tlv_stream.write(&mut metadata_material.hmac).unwrap();
+                               (Metadata::Bytes(metadata_material.derive_metadata()), None)
+                       },
+                       Metadata::DerivedSigningPubkey(mut metadata_material) => {
+                               tlv_stream.write(&mut metadata_material.hmac).unwrap();
+                               let secp_ctx = secp_ctx.unwrap();
+                               let (metadata, keys) = metadata_material.derive_metadata_and_keys(secp_ctx);
+                               (Metadata::Bytes(metadata), Some(keys))
+                       },
+               }
+       }
+}
+
+impl Default for Metadata {
+       fn default() -> Self {
+               Metadata::Bytes(vec![])
+       }
+}
+
+impl fmt::Debug for Metadata {
+       fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+               match self {
+                       Metadata::Bytes(bytes) => bytes.fmt(f),
+                       Metadata::Derived(_) => f.write_str("Derived"),
+                       Metadata::DerivedSigningPubkey(_) => f.write_str("DerivedSigningPubkey"),
+               }
+       }
+}
+
+#[cfg(test)]
+impl PartialEq for Metadata {
+       fn eq(&self, other: &Self) -> bool {
+               match self {
+                       Metadata::Bytes(bytes) => if let Metadata::Bytes(other_bytes) = other {
+                               bytes == other_bytes
+                       } else {
+                               false
+                       },
+                       Metadata::Derived(_) => false,
+                       Metadata::DerivedSigningPubkey(_) => false,
+               }
+       }
+}
+
+/// Material used to create metadata for a message.
+#[derive(Clone)]
+pub(super) struct MetadataMaterial {
+       nonce: Nonce,
+       hmac: HmacEngine<Sha256>,
+}
+
+impl MetadataMaterial {
+       pub fn new(nonce: Nonce, expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN]) -> Self {
+               Self {
+                       nonce,
+                       hmac: expanded_key.hmac_for_offer(nonce, iv_bytes),
+               }
+       }
+
+       fn derive_metadata(mut self) -> Vec<u8> {
+               self.hmac.input(DERIVED_METADATA_HMAC_INPUT);
+
+               let mut bytes = self.nonce.as_slice().to_vec();
+               bytes.extend_from_slice(&Hmac::from_engine(self.hmac).into_inner());
+               bytes
+       }
+
+       fn derive_metadata_and_keys<T: secp256k1::Signing>(
+               mut self, secp_ctx: &Secp256k1<T>
+       ) -> (Vec<u8>, KeyPair) {
+               self.hmac.input(DERIVED_METADATA_AND_KEYS_HMAC_INPUT);
+
+               let hmac = Hmac::from_engine(self.hmac);
+               let privkey = SecretKey::from_slice(hmac.as_inner()).unwrap();
+               let keys = KeyPair::from_secret_key(secp_ctx, &privkey);
+               (self.nonce.as_slice().to_vec(), keys)
+       }
+}
+
+pub(super) fn derive_keys(nonce: Nonce, expanded_key: &ExpandedKey) -> KeyPair {
+       const IV_BYTES: &[u8; IV_LEN] = b"LDK Invoice ~~~~";
+       let secp_ctx = Secp256k1::new();
+       let hmac = Hmac::from_engine(expanded_key.hmac_for_offer(nonce, IV_BYTES));
+       let privkey = SecretKey::from_slice(hmac.as_inner()).unwrap();
+       KeyPair::from_secret_key(&secp_ctx, &privkey)
+}
+
+/// Verifies data given in a TLV stream was used to produce the given metadata, consisting of:
+/// - a 128-bit [`Nonce`] and possibly
+/// - a [`Sha256`] hash of the nonce and the TLV records using the [`ExpandedKey`].
+///
+/// If the latter is not included in the metadata, the TLV stream is used to check if the given
+/// `signing_pubkey` can be derived from it.
+pub(super) fn verify_metadata<'a, T: secp256k1::Signing>(
+       metadata: &[u8], expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN],
+       signing_pubkey: PublicKey, tlv_stream: impl core::iter::Iterator<Item = TlvRecord<'a>>,
+       secp_ctx: &Secp256k1<T>
+) -> Result<Option<KeyPair>, ()> {
+       let hmac = hmac_for_message(metadata, expanded_key, iv_bytes, tlv_stream)?;
+
+       if metadata.len() == Nonce::LENGTH {
+               let derived_keys = KeyPair::from_secret_key(
+                       secp_ctx, &SecretKey::from_slice(hmac.as_inner()).unwrap()
+               );
+               if fixed_time_eq(&signing_pubkey.serialize(), &derived_keys.public_key().serialize()) {
+                       Ok(Some(derived_keys))
+               } else {
+                       Err(())
+               }
+       } else if metadata[Nonce::LENGTH..].len() == Sha256::LEN {
+               if fixed_time_eq(&metadata[Nonce::LENGTH..], &hmac.into_inner()) {
+                       Ok(None)
+               } else {
+                       Err(())
+               }
+       } else {
+               Err(())
+       }
+}
+
+fn hmac_for_message<'a>(
+       metadata: &[u8], expanded_key: &ExpandedKey, iv_bytes: &[u8; IV_LEN],
+       tlv_stream: impl core::iter::Iterator<Item = TlvRecord<'a>>
+) -> Result<Hmac<Sha256>, ()> {
+       if metadata.len() < Nonce::LENGTH {
+               return Err(());
+       }
+
+       let nonce = match Nonce::try_from(&metadata[..Nonce::LENGTH]) {
+               Ok(nonce) => nonce,
+               Err(_) => return Err(()),
+       };
+       let mut hmac = expanded_key.hmac_for_offer(nonce, iv_bytes);
+
+       for record in tlv_stream {
+               hmac.input(record.record_bytes);
+       }
+
+       if metadata.len() == Nonce::LENGTH {
+               hmac.input(DERIVED_METADATA_AND_KEYS_HMAC_INPUT);
+       } else {
+               hmac.input(DERIVED_METADATA_HMAC_INPUT);
+       }
+
+       Ok(Hmac::from_engine(hmac))
+}
diff --git a/lightning/src/offers/test_utils.rs b/lightning/src/offers/test_utils.rs
new file mode 100644 (file)
index 0000000..8ded4a6
--- /dev/null
@@ -0,0 +1,119 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for testing BOLT 12 Offers interfaces
+
+use bitcoin::secp256k1::{KeyPair, Message, PublicKey, Secp256k1, SecretKey};
+use bitcoin::secp256k1::schnorr::Signature;
+use core::convert::Infallible;
+use core::time::Duration;
+use crate::blinded_path::{BlindedHop, BlindedPath};
+use crate::chain::keysinterface::EntropySource;
+use crate::ln::PaymentHash;
+use crate::ln::features::BlindedHopFeatures;
+use crate::offers::invoice::BlindedPayInfo;
+
+pub(super) fn payer_keys() -> KeyPair {
+       let secp_ctx = Secp256k1::new();
+       KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap())
+}
+
+pub(super) fn payer_sign(digest: &Message) -> Result<Signature, Infallible> {
+       let secp_ctx = Secp256k1::new();
+       let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32]).unwrap());
+       Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
+}
+
+pub(super) fn payer_pubkey() -> PublicKey {
+       payer_keys().public_key()
+}
+
+pub(super) fn recipient_keys() -> KeyPair {
+       let secp_ctx = Secp256k1::new();
+       KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap())
+}
+
+pub(super) fn recipient_sign(digest: &Message) -> Result<Signature, Infallible> {
+       let secp_ctx = Secp256k1::new();
+       let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[43; 32]).unwrap());
+       Ok(secp_ctx.sign_schnorr_no_aux_rand(digest, &keys))
+}
+
+pub(super) fn recipient_pubkey() -> PublicKey {
+       recipient_keys().public_key()
+}
+
+pub(super) fn pubkey(byte: u8) -> PublicKey {
+       let secp_ctx = Secp256k1::new();
+       PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
+}
+
+pub(super) fn privkey(byte: u8) -> SecretKey {
+       SecretKey::from_slice(&[byte; 32]).unwrap()
+}
+
+pub(super) fn payment_paths() -> Vec<(BlindedPath, BlindedPayInfo)> {
+       let paths = vec![
+               BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                               BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
+                       ],
+               },
+               BlindedPath {
+                       introduction_node_id: pubkey(40),
+                       blinding_point: pubkey(41),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
+                               BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
+                       ],
+               },
+       ];
+
+       let payinfo = vec![
+               BlindedPayInfo {
+                       fee_base_msat: 1,
+                       fee_proportional_millionths: 1_000,
+                       cltv_expiry_delta: 42,
+                       htlc_minimum_msat: 100,
+                       htlc_maximum_msat: 1_000_000_000_000,
+                       features: BlindedHopFeatures::empty(),
+               },
+               BlindedPayInfo {
+                       fee_base_msat: 1,
+                       fee_proportional_millionths: 1_000,
+                       cltv_expiry_delta: 42,
+                       htlc_minimum_msat: 100,
+                       htlc_maximum_msat: 1_000_000_000_000,
+                       features: BlindedHopFeatures::empty(),
+               },
+       ];
+
+       paths.into_iter().zip(payinfo.into_iter()).collect()
+}
+
+pub(super) fn payment_hash() -> PaymentHash {
+       PaymentHash([42; 32])
+}
+
+pub(super) fn now() -> Duration {
+       std::time::SystemTime::now()
+               .duration_since(std::time::SystemTime::UNIX_EPOCH)
+               .expect("SystemTime::now() should come after SystemTime::UNIX_EPOCH")
+}
+
+pub(super) struct FixedEntropy;
+
+impl EntropySource for FixedEntropy {
+       fn get_secure_random_bytes(&self) -> [u8; 32] {
+               [42; 32]
+       }
+}
diff --git a/lightning/src/onion_message/blinded_path.rs b/lightning/src/onion_message/blinded_path.rs
deleted file mode 100644 (file)
index 946e5e7..0000000
+++ /dev/null
@@ -1,231 +0,0 @@
-// This file is Copyright its original authors, visible in version control
-// history.
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-//! Creating blinded paths and related utilities live here.
-
-use bitcoin::hashes::{Hash, HashEngine};
-use bitcoin::hashes::sha256::Hash as Sha256;
-use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey};
-
-use crate::chain::keysinterface::{EntropySource, NodeSigner, Recipient};
-use super::packet::ControlTlvs;
-use super::utils;
-use crate::ln::msgs::DecodeError;
-use crate::ln::onion_utils;
-use crate::util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter};
-use crate::util::ser::{FixedLengthReader, LengthReadableArgs, Readable, VecWriter, Writeable, Writer};
-
-use core::mem;
-use core::ops::Deref;
-use crate::io::{self, Cursor};
-use crate::prelude::*;
-
-/// Onion messages can be sent and received to blinded paths, which serve to hide the identity of
-/// the recipient.
-#[derive(Clone, Debug, PartialEq)]
-pub struct BlindedPath {
-       /// To send to a blinded path, the sender first finds a route to the unblinded
-       /// `introduction_node_id`, which can unblind its [`encrypted_payload`] to find out the onion
-       /// message's next hop and forward it along.
-       ///
-       /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(crate) introduction_node_id: PublicKey,
-       /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion
-       /// message.
-       ///
-       /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(crate) blinding_point: PublicKey,
-       /// The hops composing the blinded path.
-       pub(crate) blinded_hops: Vec<BlindedHop>,
-}
-
-/// Used to construct the blinded hops portion of a blinded path. These hops cannot be identified
-/// by outside observers and thus can be used to hide the identity of the recipient.
-#[derive(Clone, Debug, PartialEq)]
-pub struct BlindedHop {
-       /// The blinded node id of this hop in a blinded path.
-       pub(crate) blinded_node_id: PublicKey,
-       /// The encrypted payload intended for this hop in a blinded path.
-       // The node sending to this blinded path will later encode this payload into the onion packet for
-       // this hop.
-       pub(crate) encrypted_payload: Vec<u8>,
-}
-
-impl BlindedPath {
-       /// Create a blinded path to be forwarded along `node_pks`. The last node pubkey in `node_pks`
-       /// will be the destination node.
-       ///
-       /// Errors if less than two hops are provided or if `node_pk`(s) are invalid.
-       //  TODO: make all payloads the same size with padding + add dummy hops
-       pub fn new<ES: EntropySource, T: secp256k1::Signing + secp256k1::Verification>
-               (node_pks: &[PublicKey], entropy_source: &ES, secp_ctx: &Secp256k1<T>) -> Result<Self, ()>
-       {
-               if node_pks.len() < 2 { return Err(()) }
-               let blinding_secret_bytes = entropy_source.get_secure_random_bytes();
-               let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted");
-               let introduction_node_id = node_pks[0];
-
-               Ok(BlindedPath {
-                       introduction_node_id,
-                       blinding_point: PublicKey::from_secret_key(secp_ctx, &blinding_secret),
-                       blinded_hops: blinded_hops(secp_ctx, node_pks, &blinding_secret).map_err(|_| ())?,
-               })
-       }
-
-       // Advance the blinded path by one hop, so make the second hop into the new introduction node.
-       pub(super) fn advance_by_one<NS: Deref, T: secp256k1::Signing + secp256k1::Verification>
-               (&mut self, node_signer: &NS, secp_ctx: &Secp256k1<T>) -> Result<(), ()>
-               where NS::Target: NodeSigner
-       {
-               let control_tlvs_ss = node_signer.ecdh(Recipient::Node, &self.blinding_point, None)?;
-               let rho = onion_utils::gen_rho_from_shared_secret(&control_tlvs_ss.secret_bytes());
-               let encrypted_control_tlvs = self.blinded_hops.remove(0).encrypted_payload;
-               let mut s = Cursor::new(&encrypted_control_tlvs);
-               let mut reader = FixedLengthReader::new(&mut s, encrypted_control_tlvs.len() as u64);
-               match ChaChaPolyReadAdapter::read(&mut reader, rho) {
-                       Ok(ChaChaPolyReadAdapter { readable: ControlTlvs::Forward(ForwardTlvs {
-                               mut next_node_id, next_blinding_override,
-                       })}) => {
-                               let mut new_blinding_point = match next_blinding_override {
-                                       Some(blinding_point) => blinding_point,
-                                       None => {
-                                               let blinding_factor = {
-                                                       let mut sha = Sha256::engine();
-                                                       sha.input(&self.blinding_point.serialize()[..]);
-                                                       sha.input(control_tlvs_ss.as_ref());
-                                                       Sha256::from_engine(sha).into_inner()
-                                               };
-                                               self.blinding_point.mul_tweak(secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap())
-                                                       .map_err(|_| ())?
-                                       }
-                               };
-                               mem::swap(&mut self.blinding_point, &mut new_blinding_point);
-                               mem::swap(&mut self.introduction_node_id, &mut next_node_id);
-                               Ok(())
-                       },
-                       _ => Err(())
-               }
-       }
-}
-
-/// Construct blinded hops for the given `unblinded_path`.
-fn blinded_hops<T: secp256k1::Signing + secp256k1::Verification>(
-       secp_ctx: &Secp256k1<T>, unblinded_path: &[PublicKey], session_priv: &SecretKey
-) -> Result<Vec<BlindedHop>, secp256k1::Error> {
-       let mut blinded_hops = Vec::with_capacity(unblinded_path.len());
-
-       let mut prev_ss_and_blinded_node_id = None;
-       utils::construct_keys_callback(secp_ctx, unblinded_path, None, session_priv, |blinded_node_id, _, _, encrypted_payload_ss, unblinded_pk, _| {
-               if let Some((prev_ss, prev_blinded_node_id)) = prev_ss_and_blinded_node_id {
-                       if let Some(pk) = unblinded_pk {
-                               let payload = ForwardTlvs {
-                                       next_node_id: pk,
-                                       next_blinding_override: None,
-                               };
-                               blinded_hops.push(BlindedHop {
-                                       blinded_node_id: prev_blinded_node_id,
-                                       encrypted_payload: encrypt_payload(payload, prev_ss),
-                               });
-                       } else { debug_assert!(false); }
-               }
-               prev_ss_and_blinded_node_id = Some((encrypted_payload_ss, blinded_node_id));
-       })?;
-
-       if let Some((final_ss, final_blinded_node_id)) = prev_ss_and_blinded_node_id {
-               let final_payload = ReceiveTlvs { path_id: None };
-               blinded_hops.push(BlindedHop {
-                       blinded_node_id: final_blinded_node_id,
-                       encrypted_payload: encrypt_payload(final_payload, final_ss),
-               });
-       } else { debug_assert!(false) }
-
-       Ok(blinded_hops)
-}
-
-/// Encrypt TLV payload to be used as a [`BlindedHop::encrypted_payload`].
-fn encrypt_payload<P: Writeable>(payload: P, encrypted_tlvs_ss: [u8; 32]) -> Vec<u8> {
-       let mut writer = VecWriter(Vec::new());
-       let write_adapter = ChaChaPolyWriteAdapter::new(encrypted_tlvs_ss, &payload);
-       write_adapter.write(&mut writer).expect("In-memory writes cannot fail");
-       writer.0
-}
-
-impl Writeable for BlindedPath {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               self.introduction_node_id.write(w)?;
-               self.blinding_point.write(w)?;
-               (self.blinded_hops.len() as u8).write(w)?;
-               for hop in &self.blinded_hops {
-                       hop.write(w)?;
-               }
-               Ok(())
-       }
-}
-
-impl Readable for BlindedPath {
-       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let introduction_node_id = Readable::read(r)?;
-               let blinding_point = Readable::read(r)?;
-               let num_hops: u8 = Readable::read(r)?;
-               if num_hops == 0 { return Err(DecodeError::InvalidValue) }
-               let mut blinded_hops: Vec<BlindedHop> = Vec::with_capacity(num_hops.into());
-               for _ in 0..num_hops {
-                       blinded_hops.push(Readable::read(r)?);
-               }
-               Ok(BlindedPath {
-                       introduction_node_id,
-                       blinding_point,
-                       blinded_hops,
-               })
-       }
-}
-
-impl_writeable!(BlindedHop, {
-       blinded_node_id,
-       encrypted_payload
-});
-
-/// TLVs to encode in an intermediate onion message packet's hop data. When provided in a blinded
-/// route, they are encoded into [`BlindedHop::encrypted_payload`].
-pub(crate) struct ForwardTlvs {
-       /// The node id of the next hop in the onion message's path.
-       pub(super) next_node_id: PublicKey,
-       /// Senders to a blinded path use this value to concatenate the route they find to the
-       /// introduction node with the blinded path.
-       pub(super) next_blinding_override: Option<PublicKey>,
-}
-
-/// Similar to [`ForwardTlvs`], but these TLVs are for the final node.
-pub(crate) struct ReceiveTlvs {
-       /// If `path_id` is `Some`, it is used to identify the blinded path that this onion message is
-       /// sending to. This is useful for receivers to check that said blinded path is being used in
-       /// the right context.
-       pub(super) path_id: Option<[u8; 32]>,
-}
-
-impl Writeable for ForwardTlvs {
-       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-               // TODO: write padding
-               encode_tlv_stream!(writer, {
-                       (4, self.next_node_id, required),
-                       (8, self.next_blinding_override, option)
-               });
-               Ok(())
-       }
-}
-
-impl Writeable for ReceiveTlvs {
-       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-               // TODO: write padding
-               encode_tlv_stream!(writer, {
-                       (6, self.path_id, option),
-               });
-               Ok(())
-       }
-}
index 89582305c2107baabcce522937ab13f793b0efbc..991b4e9e7df7666eabca6273926702ea435d2a9b 100644 (file)
@@ -9,10 +9,11 @@
 
 //! Onion message testing and test utilities live here.
 
+use crate::blinded_path::BlindedPath;
 use crate::chain::keysinterface::{NodeSigner, Recipient};
 use crate::ln::features::InitFeatures;
 use crate::ln::msgs::{self, DecodeError, OnionMessageHandler};
-use super::{BlindedPath, CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError};
+use super::{CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError};
 use crate::util::ser::{Writeable, Writer};
 use crate::util::test_utils;
 
@@ -135,7 +136,7 @@ fn two_unblinded_two_blinded() {
        let test_msg = OnionMessageContents::Custom(TestCustomMessage {});
 
        let secp_ctx = Secp256k1::new();
-       let blinded_path = BlindedPath::new(&[nodes[3].get_node_pk(), nodes[4].get_node_pk()], &*nodes[4].keys_manager, &secp_ctx).unwrap();
+       let blinded_path = BlindedPath::new_for_message(&[nodes[3].get_node_pk(), nodes[4].get_node_pk()], &*nodes[4].keys_manager, &secp_ctx).unwrap();
 
        nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::BlindedPath(blinded_path), test_msg, None).unwrap();
        pass_along_path(&nodes, None);
@@ -147,7 +148,7 @@ fn three_blinded_hops() {
        let test_msg = OnionMessageContents::Custom(TestCustomMessage {});
 
        let secp_ctx = Secp256k1::new();
-       let blinded_path = BlindedPath::new(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap();
+       let blinded_path = BlindedPath::new_for_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap();
 
        nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), test_msg, None).unwrap();
        pass_along_path(&nodes, None);
@@ -173,13 +174,13 @@ fn we_are_intro_node() {
        let test_msg = TestCustomMessage {};
 
        let secp_ctx = Secp256k1::new();
-       let blinded_path = BlindedPath::new(&[nodes[0].get_node_pk(), nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
+       let blinded_path = BlindedPath::new_for_message(&[nodes[0].get_node_pk(), nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
 
        nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), OnionMessageContents::Custom(test_msg.clone()), None).unwrap();
        pass_along_path(&nodes, None);
 
        // Try with a two-hop blinded path where we are the introduction node.
-       let blinded_path = BlindedPath::new(&[nodes[0].get_node_pk(), nodes[1].get_node_pk()], &*nodes[1].keys_manager, &secp_ctx).unwrap();
+       let blinded_path = BlindedPath::new_for_message(&[nodes[0].get_node_pk(), nodes[1].get_node_pk()], &*nodes[1].keys_manager, &secp_ctx).unwrap();
        nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), OnionMessageContents::Custom(test_msg), None).unwrap();
        nodes.remove(2);
        pass_along_path(&nodes, None);
@@ -193,13 +194,13 @@ fn invalid_blinded_path_error() {
 
        // 0 hops
        let secp_ctx = Secp256k1::new();
-       let mut blinded_path = BlindedPath::new(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
+       let mut blinded_path = BlindedPath::new_for_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
        blinded_path.blinded_hops.clear();
        let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), OnionMessageContents::Custom(test_msg.clone()), None).unwrap_err();
        assert_eq!(err, SendError::TooFewBlindedHops);
 
        // 1 hop
-       let mut blinded_path = BlindedPath::new(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
+       let mut blinded_path = BlindedPath::new_for_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
        blinded_path.blinded_hops.remove(0);
        assert_eq!(blinded_path.blinded_hops.len(), 1);
        let err = nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), OnionMessageContents::Custom(test_msg), None).unwrap_err();
@@ -213,7 +214,7 @@ fn reply_path() {
        let secp_ctx = Secp256k1::new();
 
        // Destination::Node
-       let reply_path = BlindedPath::new(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap();
+       let reply_path = BlindedPath::new_for_message(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap();
        nodes[0].messenger.send_onion_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk()], Destination::Node(nodes[3].get_node_pk()), OnionMessageContents::Custom(test_msg.clone()), Some(reply_path)).unwrap();
        pass_along_path(&nodes, None);
        // Make sure the last node successfully decoded the reply path.
@@ -222,8 +223,8 @@ fn reply_path() {
                &format!("Received an onion message with path_id None and a reply_path"), 1);
 
        // Destination::BlindedPath
-       let blinded_path = BlindedPath::new(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap();
-       let reply_path = BlindedPath::new(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap();
+       let blinded_path = BlindedPath::new_for_message(&[nodes[1].get_node_pk(), nodes[2].get_node_pk(), nodes[3].get_node_pk()], &*nodes[3].keys_manager, &secp_ctx).unwrap();
+       let reply_path = BlindedPath::new_for_message(&[nodes[2].get_node_pk(), nodes[1].get_node_pk(), nodes[0].get_node_pk()], &*nodes[0].keys_manager, &secp_ctx).unwrap();
 
        nodes[0].messenger.send_onion_message(&[], Destination::BlindedPath(blinded_path), OnionMessageContents::Custom(test_msg), Some(reply_path)).unwrap();
        pass_along_path(&nodes, None);
index b9269c7cf2c058dc9297b451f029fb068fa06793..b50282433b18a128df1185711861a98ea0b44aa4 100644 (file)
@@ -15,16 +15,15 @@ use bitcoin::hashes::hmac::{Hmac, HmacEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey};
 
+use crate::blinded_path::{BlindedPath, ForwardTlvs, ReceiveTlvs, utils};
 use crate::chain::keysinterface::{EntropySource, KeysManager, NodeSigner, Recipient};
 use crate::events::OnionMessageProvider;
 use crate::ln::features::{InitFeatures, NodeFeatures};
 use crate::ln::msgs::{self, OnionMessageHandler};
 use crate::ln::onion_utils;
 use crate::ln::peer_handler::IgnoringMessageHandler;
-use super::blinded_path::{BlindedPath, ForwardTlvs, ReceiveTlvs};
 pub use super::packet::{CustomOnionMessageContents, OnionMessageContents};
 use super::packet::{BIG_PACKET_HOP_DATA_LEN, ForwardControlTlvs, Packet, Payload, ReceiveControlTlvs, SMALL_PACKET_HOP_DATA_LEN};
-use super::utils;
 use crate::util::logger::Logger;
 use crate::util::ser::Writeable;
 
@@ -43,9 +42,10 @@ use crate::prelude::*;
 /// # extern crate bitcoin;
 /// # use bitcoin::hashes::_export::_core::time::Duration;
 /// # use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
+/// # use lightning::blinded_path::BlindedPath;
 /// # use lightning::chain::keysinterface::KeysManager;
 /// # use lightning::ln::peer_handler::IgnoringMessageHandler;
-/// # use lightning::onion_message::{BlindedPath, CustomOnionMessageContents, Destination, OnionMessageContents, OnionMessenger};
+/// # use lightning::onion_message::{CustomOnionMessageContents, Destination, OnionMessageContents, OnionMessenger};
 /// # use lightning::util::logger::{Logger, Record};
 /// # use lightning::util::ser::{Writeable, Writer};
 /// # use lightning::io;
@@ -91,7 +91,7 @@ use crate::prelude::*;
 /// // Create a blinded path to yourself, for someone to send an onion message to.
 /// # let your_node_id = hop_node_id1;
 /// let hops = [hop_node_id3, hop_node_id4, your_node_id];
-/// let blinded_path = BlindedPath::new(&hops, &keys_manager, &secp_ctx).unwrap();
+/// let blinded_path = BlindedPath::new_for_message(&hops, &keys_manager, &secp_ctx).unwrap();
 ///
 /// // Send a custom onion message to a blinded path.
 /// # let intermediate_hops = [hop_node_id1, hop_node_id2];
@@ -226,7 +226,7 @@ impl<ES: Deref, NS: Deref, L: Deref, CMH: Deref> OnionMessenger<ES, NS, L, CMH>
                                let our_node_id = self.node_signer.get_node_id(Recipient::Node)
                                        .map_err(|()| SendError::GetNodeIdFailed)?;
                                if blinded_path.introduction_node_id == our_node_id {
-                                       blinded_path.advance_by_one(&self.node_signer, &self.secp_ctx)
+                                       blinded_path.advance_message_path_by_one(&self.node_signer, &self.secp_ctx)
                                                .map_err(|()| SendError::BlindedPathAdvanceFailed)?;
                                }
                        }
index e735b428ea03dea6a26018bf23ac8ff4670e7c33..713b83c62d67d3e0a2fdcc39fd5a004d8a744e6c 100644 (file)
 //! information on its usage.
 //!
 //! [offers]: <https://github.com/lightning/bolts/pull/798>
-//! [blinded paths]: crate::onion_message::BlindedPath
+//! [blinded paths]: crate::blinded_path::BlindedPath
 
-mod blinded_path;
 mod messenger;
 mod packet;
-mod utils;
 #[cfg(test)]
 mod functional_tests;
 
 // Re-export structs so they can be imported with just the `onion_message::` module prefix.
-pub use self::blinded_path::{BlindedPath, BlindedHop};
 pub use self::messenger::{CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError, SimpleArcOnionMessenger, SimpleRefOnionMessenger};
-pub(crate) use self::packet::Packet;
+pub(crate) use self::packet::{ControlTlvs, Packet};
index d22ff0682da285ed5d717158060c72144f70b533..2fb2407dbdd093bdbe4ae260e31f12fbe4722d8a 100644 (file)
@@ -12,9 +12,9 @@
 use bitcoin::secp256k1::PublicKey;
 use bitcoin::secp256k1::ecdh::SharedSecret;
 
+use crate::blinded_path::{BlindedPath, ForwardTlvs, ReceiveTlvs};
 use crate::ln::msgs::DecodeError;
 use crate::ln::onion_utils;
-use super::blinded_path::{BlindedPath, ForwardTlvs, ReceiveTlvs};
 use super::messenger::CustomOnionMessageHandler;
 use crate::util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter};
 use crate::util::ser::{BigSize, FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer};
@@ -149,7 +149,7 @@ pub(super) enum ForwardControlTlvs {
        Blinded(Vec<u8>),
        /// If we're constructing an onion message hop through an intermediate unblinded node, we'll need
        /// to construct the intermediate hop's control TLVs in their unblinded state to avoid encoding
-       /// them into an intermediate Vec. See [`super::blinded_path::ForwardTlvs`] for more info.
+       /// them into an intermediate Vec. See [`crate::blinded_path::ForwardTlvs`] for more info.
        Unblinded(ForwardTlvs),
 }
 
@@ -157,7 +157,7 @@ pub(super) enum ForwardControlTlvs {
 pub(super) enum ReceiveControlTlvs {
        /// See [`ForwardControlTlvs::Blinded`].
        Blinded(Vec<u8>),
-       /// See [`ForwardControlTlvs::Unblinded`] and [`super::blinded_path::ReceiveTlvs`].
+       /// See [`ForwardControlTlvs::Unblinded`] and [`crate::blinded_path::ReceiveTlvs`].
        Unblinded(ReceiveTlvs),
 }
 
@@ -255,7 +255,7 @@ impl<H: CustomOnionMessageHandler> ReadableArgs<(SharedSecret, &H)> for Payload<
 /// When reading a packet off the wire, we don't know a priori whether the packet is to be forwarded
 /// or received. Thus we read a ControlTlvs rather than reading a ForwardControlTlvs or
 /// ReceiveControlTlvs directly.
-pub(super) enum ControlTlvs {
+pub(crate) enum ControlTlvs {
        /// This onion message is intended to be forwarded.
        Forward(ForwardTlvs),
        /// This onion message is intended to be received.
diff --git a/lightning/src/onion_message/utils.rs b/lightning/src/onion_message/utils.rs
deleted file mode 100644 (file)
index ae9e0a7..0000000
+++ /dev/null
@@ -1,98 +0,0 @@
-// This file is Copyright its original authors, visible in version control
-// history.
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-//! Onion message utility methods live here.
-
-use bitcoin::hashes::{Hash, HashEngine};
-use bitcoin::hashes::hmac::{Hmac, HmacEngine};
-use bitcoin::hashes::sha256::Hash as Sha256;
-use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey, Scalar};
-use bitcoin::secp256k1::ecdh::SharedSecret;
-
-use crate::ln::onion_utils;
-use super::blinded_path::BlindedPath;
-use super::messenger::Destination;
-
-use crate::prelude::*;
-
-// TODO: DRY with onion_utils::construct_onion_keys_callback
-#[inline]
-pub(super) fn construct_keys_callback<T: secp256k1::Signing + secp256k1::Verification,
-       FType: FnMut(PublicKey, SharedSecret, PublicKey, [u8; 32], Option<PublicKey>, Option<Vec<u8>>)>(
-       secp_ctx: &Secp256k1<T>, unblinded_path: &[PublicKey], destination: Option<Destination>,
-       session_priv: &SecretKey, mut callback: FType
-) -> Result<(), secp256k1::Error> {
-       let mut msg_blinding_point_priv = session_priv.clone();
-       let mut msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv);
-       let mut onion_packet_pubkey_priv = msg_blinding_point_priv.clone();
-       let mut onion_packet_pubkey = msg_blinding_point.clone();
-
-       macro_rules! build_keys {
-               ($pk: expr, $blinded: expr, $encrypted_payload: expr) => {{
-                       let encrypted_data_ss = SharedSecret::new(&$pk, &msg_blinding_point_priv);
-
-                       let blinded_hop_pk = if $blinded { $pk } else {
-                               let hop_pk_blinding_factor = {
-                                       let mut hmac = HmacEngine::<Sha256>::new(b"blinded_node_id");
-                                       hmac.input(encrypted_data_ss.as_ref());
-                                       Hmac::from_engine(hmac).into_inner()
-                               };
-                               $pk.mul_tweak(secp_ctx, &Scalar::from_be_bytes(hop_pk_blinding_factor).unwrap())?
-                       };
-                       let onion_packet_ss = SharedSecret::new(&blinded_hop_pk, &onion_packet_pubkey_priv);
-
-                       let rho = onion_utils::gen_rho_from_shared_secret(encrypted_data_ss.as_ref());
-                       let unblinded_pk_opt = if $blinded { None } else { Some($pk) };
-                       callback(blinded_hop_pk, onion_packet_ss, onion_packet_pubkey, rho, unblinded_pk_opt, $encrypted_payload);
-                       (encrypted_data_ss, onion_packet_ss)
-               }}
-       }
-
-       macro_rules! build_keys_in_loop {
-               ($pk: expr, $blinded: expr, $encrypted_payload: expr) => {
-                       let (encrypted_data_ss, onion_packet_ss) = build_keys!($pk, $blinded, $encrypted_payload);
-
-                       let msg_blinding_point_blinding_factor = {
-                               let mut sha = Sha256::engine();
-                               sha.input(&msg_blinding_point.serialize()[..]);
-                               sha.input(encrypted_data_ss.as_ref());
-                               Sha256::from_engine(sha).into_inner()
-                       };
-
-                       msg_blinding_point_priv = msg_blinding_point_priv.mul_tweak(&Scalar::from_be_bytes(msg_blinding_point_blinding_factor).unwrap())?;
-                       msg_blinding_point = PublicKey::from_secret_key(secp_ctx, &msg_blinding_point_priv);
-
-                       let onion_packet_pubkey_blinding_factor = {
-                               let mut sha = Sha256::engine();
-                               sha.input(&onion_packet_pubkey.serialize()[..]);
-                               sha.input(onion_packet_ss.as_ref());
-                               Sha256::from_engine(sha).into_inner()
-                       };
-                       onion_packet_pubkey_priv = onion_packet_pubkey_priv.mul_tweak(&Scalar::from_be_bytes(onion_packet_pubkey_blinding_factor).unwrap())?;
-                       onion_packet_pubkey = PublicKey::from_secret_key(secp_ctx, &onion_packet_pubkey_priv);
-               };
-       }
-
-       for pk in unblinded_path {
-               build_keys_in_loop!(*pk, false, None);
-       }
-       if let Some(dest) = destination {
-               match dest {
-                       Destination::Node(pk) => {
-                               build_keys!(pk, false, None);
-                       },
-                       Destination::BlindedPath(BlindedPath { blinded_hops, .. }) => {
-                               for hop in blinded_hops {
-                                       build_keys_in_loop!(hop.blinded_node_id, true, Some(hop.encrypted_payload));
-                               }
-                       },
-               }
-       }
-       Ok(())
-}
index 892d1b40a962ca9f2f6d80fd840eb4fd164637cc..11bf532e90fc40388f0debf30968fa5c6cb813bd 100644 (file)
@@ -16,6 +16,7 @@ use bitcoin::secp256k1;
 
 use bitcoin::hashes::sha256d::Hash as Sha256dHash;
 use bitcoin::hashes::Hash;
+use bitcoin::hashes::hex::FromHex;
 use bitcoin::hash_types::BlockHash;
 
 use bitcoin::network::constants::Network;
@@ -27,7 +28,7 @@ use crate::ln::msgs::{DecodeError, ErrorAction, Init, LightningError, RoutingMes
 use crate::ln::msgs::{ChannelAnnouncement, ChannelUpdate, NodeAnnouncement, GossipTimestampFilter};
 use crate::ln::msgs::{QueryChannelRange, ReplyChannelRange, QueryShortChannelIds, ReplyShortChannelIdsEnd};
 use crate::ln::msgs;
-use crate::routing::utxo::{self, UtxoLookup};
+use crate::routing::utxo::{self, UtxoLookup, UtxoResolver};
 use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, MaybeReadable};
 use crate::util::logger::{Logger, Level};
 use crate::util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK};
@@ -38,11 +39,13 @@ use crate::io;
 use crate::io_extras::{copy, sink};
 use crate::prelude::*;
 use core::{cmp, fmt};
+use core::convert::TryFrom;
 use crate::sync::{RwLock, RwLockReadGuard};
 #[cfg(feature = "std")]
 use core::sync::atomic::{AtomicUsize, Ordering};
 use crate::sync::Mutex;
 use core::ops::{Bound, Deref};
+use core::str::FromStr;
 
 #[cfg(feature = "std")]
 use std::time::{SystemTime, UNIX_EPOCH};
@@ -76,6 +79,11 @@ impl NodeId {
        pub fn as_slice(&self) -> &[u8] {
                &self.0
        }
+
+       /// Get the public key from this NodeId
+       pub fn as_pubkey(&self) -> Result<PublicKey, secp256k1::Error> {
+               PublicKey::from_slice(&self.0)
+       }
 }
 
 impl fmt::Debug for NodeId {
@@ -130,6 +138,29 @@ impl Readable for NodeId {
        }
 }
 
+impl From<PublicKey> for NodeId {
+       fn from(pubkey: PublicKey) -> Self {
+               Self::from_pubkey(&pubkey)
+       }
+}
+
+impl TryFrom<NodeId> for PublicKey {
+       type Error = secp256k1::Error;
+
+       fn try_from(node_id: NodeId) -> Result<Self, Self::Error> {
+               node_id.as_pubkey()
+       }
+}
+
+impl FromStr for NodeId {
+       type Err = bitcoin::hashes::hex::Error;
+
+       fn from_str(s: &str) -> Result<Self, Self::Err> {
+               let data: [u8; PUBLIC_KEY_SIZE] = FromHex::from_hex(s)?;
+               Ok(NodeId(data))
+       }
+}
+
 /// Represents the network as nodes and channels between them
 pub struct NetworkGraph<L: Deref> where L::Target: Logger {
        secp_ctx: Secp256k1<secp256k1::VerifyOnly>,
@@ -181,7 +212,7 @@ pub enum NetworkUpdate {
                msg: ChannelUpdate,
        },
        /// An error indicating that a channel failed to route a payment, which should be applied via
-       /// [`NetworkGraph::channel_failed`].
+       /// [`NetworkGraph::channel_failed_permanent`] if permanent.
        ChannelFailure {
                /// The short channel id of the closed channel.
                short_channel_id: u64,
@@ -321,9 +352,10 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                                let _ = self.update_channel(msg);
                        },
                        NetworkUpdate::ChannelFailure { short_channel_id, is_permanent } => {
-                               let action = if is_permanent { "Removing" } else { "Disabling" };
-                               log_debug!(self.logger, "{} channel graph entry for {} due to a payment failure.", action, short_channel_id);
-                               self.channel_failed(short_channel_id, is_permanent);
+                               if is_permanent {
+                                       log_debug!(self.logger, "Removing channel graph entry for {} due to a payment failure.", short_channel_id);
+                                       self.channel_failed_permanent(short_channel_id);
+                               }
                        },
                        NetworkUpdate::NodeFailure { ref node_id, is_permanent } => {
                                if is_permanent {
@@ -1103,7 +1135,7 @@ impl Readable for NodeAnnouncementInfo {
 ///
 /// Since node aliases are provided by third parties, they are a potential avenue for injection
 /// attacks. Care must be taken when processing.
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub struct NodeAlias(pub [u8; 32]);
 
 impl fmt::Display for NodeAlias {
@@ -1396,7 +1428,7 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                                        features: msg.features.clone(),
                                        last_update: msg.timestamp,
                                        rgb: msg.rgb,
-                                       alias: NodeAlias(msg.alias),
+                                       alias: msg.alias,
                                        announcement_message: if should_relay { full_msg.cloned() } else { None },
                                });
 
@@ -1407,8 +1439,8 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
 
        /// Store or update channel info from a channel announcement.
        ///
-       /// You probably don't want to call this directly, instead relying on a P2PGossipSync's
-       /// RoutingMessageHandler implementation to call it indirectly. This may be useful to accept
+       /// You probably don't want to call this directly, instead relying on a [`P2PGossipSync`]'s
+       /// [`RoutingMessageHandler`] implementation to call it indirectly. This may be useful to accept
        /// routing messages from a source using a protocol other than the lightning P2P protocol.
        ///
        /// If a [`UtxoLookup`] object is provided via `utxo_lookup`, it will be called to verify
@@ -1427,6 +1459,19 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                self.update_channel_from_unsigned_announcement_intern(&msg.contents, Some(msg), utxo_lookup)
        }
 
+       /// Store or update channel info from a channel announcement.
+       ///
+       /// You probably don't want to call this directly, instead relying on a [`P2PGossipSync`]'s
+       /// [`RoutingMessageHandler`] implementation to call it indirectly. This may be useful to accept
+       /// routing messages from a source using a protocol other than the lightning P2P protocol.
+       ///
+       /// This will skip verification of if the channel is actually on-chain.
+       pub fn update_channel_from_announcement_no_lookup(
+               &self, msg: &ChannelAnnouncement
+       ) -> Result<(), LightningError> {
+               self.update_channel_from_announcement::<&UtxoResolver>(msg, &None)
+       }
+
        /// Store or update channel info from a channel announcement without verifying the associated
        /// signatures. Because we aren't given the associated signatures here we cannot relay the
        /// channel announcement to any of our peers.
@@ -1601,40 +1646,27 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                Ok(())
        }
 
-       /// Marks a channel in the graph as failed if a corresponding HTLC fail was sent.
-       /// If permanent, removes a channel from the local storage.
-       /// May cause the removal of nodes too, if this was their last channel.
-       /// If not permanent, makes channels unavailable for routing.
-       pub fn channel_failed(&self, short_channel_id: u64, is_permanent: bool) {
+       /// Marks a channel in the graph as failed permanently.
+       ///
+       /// The channel and any node for which this was their last channel are removed from the graph.
+       pub fn channel_failed_permanent(&self, short_channel_id: u64) {
                #[cfg(feature = "std")]
                let current_time_unix = Some(SystemTime::now().duration_since(UNIX_EPOCH).expect("Time must be > 1970").as_secs());
                #[cfg(not(feature = "std"))]
                let current_time_unix = None;
 
-               self.channel_failed_with_time(short_channel_id, is_permanent, current_time_unix)
+               self.channel_failed_permanent_with_time(short_channel_id, current_time_unix)
        }
 
-       /// Marks a channel in the graph as failed if a corresponding HTLC fail was sent.
-       /// If permanent, removes a channel from the local storage.
-       /// May cause the removal of nodes too, if this was their last channel.
-       /// If not permanent, makes channels unavailable for routing.
-       fn channel_failed_with_time(&self, short_channel_id: u64, is_permanent: bool, current_time_unix: Option<u64>) {
+       /// Marks a channel in the graph as failed permanently.
+       ///
+       /// The channel and any node for which this was their last channel are removed from the graph.
+       fn channel_failed_permanent_with_time(&self, short_channel_id: u64, current_time_unix: Option<u64>) {
                let mut channels = self.channels.write().unwrap();
-               if is_permanent {
-                       if let Some(chan) = channels.remove(&short_channel_id) {
-                               let mut nodes = self.nodes.write().unwrap();
-                               self.removed_channels.lock().unwrap().insert(short_channel_id, current_time_unix);
-                               Self::remove_channel_in_nodes(&mut nodes, &chan, short_channel_id);
-                       }
-               } else {
-                       if let Some(chan) = channels.get_mut(&short_channel_id) {
-                               if let Some(one_to_two) = chan.one_to_two.as_mut() {
-                                       one_to_two.enabled = false;
-                               }
-                               if let Some(two_to_one) = chan.two_to_one.as_mut() {
-                                       two_to_one.enabled = false;
-                               }
-                       }
+               if let Some(chan) = channels.remove(&short_channel_id) {
+                       let mut nodes = self.nodes.write().unwrap();
+                       self.removed_channels.lock().unwrap().insert(short_channel_id, current_time_unix);
+                       Self::remove_channel_in_nodes(&mut nodes, &chan, short_channel_id);
                }
        }
 
@@ -2029,7 +2061,7 @@ pub(crate) mod tests {
                        timestamp: 100,
                        node_id,
                        rgb: [0; 3],
-                       alias: [0; 32],
+                       alias: NodeAlias([0; 32]),
                        addresses: Vec::new(),
                        excess_address_data: Vec::new(),
                        excess_data: Vec::new(),
@@ -2419,7 +2451,7 @@ pub(crate) mod tests {
                        assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_some());
                }
 
-               // Non-permanent closing just disables a channel
+               // Non-permanent failure doesn't touch the channel at all
                {
                        match network_graph.read_only().channels().get(&short_channel_id) {
                                None => panic!(),
@@ -2436,7 +2468,7 @@ pub(crate) mod tests {
                        match network_graph.read_only().channels().get(&short_channel_id) {
                                None => panic!(),
                                Some(channel_info) => {
-                                       assert!(!channel_info.one_to_two.as_ref().unwrap().enabled);
+                                       assert!(channel_info.one_to_two.as_ref().unwrap().enabled);
                                }
                        };
                }
@@ -2570,7 +2602,7 @@ pub(crate) mod tests {
 
                        // Mark the channel as permanently failed. This will also remove the two nodes
                        // and all of the entries will be tracked as removed.
-                       network_graph.channel_failed_with_time(short_channel_id, true, Some(tracking_time));
+                       network_graph.channel_failed_permanent_with_time(short_channel_id, Some(tracking_time));
 
                        // Should not remove from tracking if insufficient time has passed
                        network_graph.remove_stale_channels_and_tracking_with_time(
@@ -2603,7 +2635,7 @@ pub(crate) mod tests {
 
                        // Mark the channel as permanently failed. This will also remove the two nodes
                        // and all of the entries will be tracked as removed.
-                       network_graph.channel_failed(short_channel_id, true);
+                       network_graph.channel_failed_permanent(short_channel_id);
 
                        // The first time we call the following, the channel will have a removal time assigned.
                        network_graph.remove_stale_channels_and_tracking_with_time(removal_time);
index 6f131fc56d6caa6aa8517cdb6cf19bad9b5dcf0a..b26298391b2f9cb2fe3561b2b5786927a27ef6a2 100644 (file)
@@ -13,10 +13,12 @@ use bitcoin::secp256k1::PublicKey;
 use bitcoin::hashes::Hash;
 use bitcoin::hashes::sha256::Hash as Sha256;
 
+use crate::blinded_path::{BlindedHop, BlindedPath};
 use crate::ln::PaymentHash;
 use crate::ln::channelmanager::{ChannelDetails, PaymentId};
 use crate::ln::features::{ChannelFeatures, InvoiceFeatures, NodeFeatures};
 use crate::ln::msgs::{DecodeError, ErrorAction, LightningError, MAX_VALUE_MSAT};
+use crate::offers::invoice::BlindedPayInfo;
 use crate::routing::gossip::{DirectedChannelInfo, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
 use crate::routing::scoring::{ChannelUsage, LockableScore, Score};
 use crate::util::ser::{Writeable, Readable, ReadableArgs, Writer};
@@ -135,19 +137,19 @@ impl<'a, S: Score> Score for ScorerAccountingForInFlightHtlcs<'a, S> {
                }
        }
 
-       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn payment_path_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.scorer.payment_path_failed(path, short_channel_id)
        }
 
-       fn payment_path_successful(&mut self, path: &[&RouteHop]) {
+       fn payment_path_successful(&mut self, path: &Path) {
                self.scorer.payment_path_successful(path)
        }
 
-       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn probe_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.scorer.probe_failed(path, short_channel_id)
        }
 
-       fn probe_successful(&mut self, path: &[&RouteHop]) {
+       fn probe_successful(&mut self, path: &Path) {
                self.scorer.probe_successful(path)
        }
 }
@@ -168,22 +170,27 @@ impl InFlightHtlcs {
        pub fn new() -> Self { InFlightHtlcs(HashMap::new()) }
 
        /// Takes in a path with payer's node id and adds the path's details to `InFlightHtlcs`.
-       pub fn process_path(&mut self, path: &[RouteHop], payer_node_id: PublicKey) {
-               if path.is_empty() { return };
+       pub fn process_path(&mut self, path: &Path, payer_node_id: PublicKey) {
+               if path.hops.is_empty() { return };
+
+               let mut cumulative_msat = 0;
+               if let Some(tail) = &path.blinded_tail {
+                       cumulative_msat += tail.final_value_msat;
+               }
+
                // total_inflight_map needs to be direction-sensitive when keeping track of the HTLC value
                // that is held up. However, the `hops` array, which is a path returned by `find_route` in
                // the router excludes the payer node. In the following lines, the payer's information is
                // hardcoded with an inflight value of 0 so that we can correctly represent the first hop
                // in our sliding window of two.
-               let reversed_hops_with_payer = path.iter().rev().skip(1)
+               let reversed_hops_with_payer = path.hops.iter().rev().skip(1)
                        .map(|hop| hop.pubkey)
                        .chain(core::iter::once(payer_node_id));
-               let mut cumulative_msat = 0;
 
                // Taking the reversed vector from above, we zip it with just the reversed hops list to
                // work "backwards" of the given path, since the last hop's `fee_msat` actually represents
                // the total amount sent.
-               for (next_hop, prev_hop) in path.iter().rev().zip(reversed_hops_with_payer) {
+               for (next_hop, prev_hop) in path.hops.iter().rev().zip(reversed_hops_with_payer) {
                        cumulative_msat += next_hop.fee_msat;
                        self.0
                                .entry((next_hop.short_channel_id, NodeId::from_pubkey(&prev_hop) < NodeId::from_pubkey(&next_hop.pubkey)))
@@ -210,7 +217,8 @@ impl Readable for InFlightHtlcs {
        }
 }
 
-/// A hop in a route
+/// A hop in a route, and additional metadata about it. "Hop" is defined as a node and the channel
+/// that leads to it.
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct RouteHop {
        /// The node_id of the node at this hop.
@@ -224,11 +232,18 @@ pub struct RouteHop {
        /// to reach this node.
        pub channel_features: ChannelFeatures,
        /// The fee taken on this hop (for paying for the use of the *next* channel in the path).
-       /// For the last hop, this should be the full value of the payment (might be more than
-       /// requested if we had to match htlc_minimum_msat).
+       /// If this is the last hop in [`Path::hops`]:
+       /// * if we're sending to a [`BlindedPath`], this is the fee paid for use of the entire blinded path
+       /// * otherwise, this is the full value of this [`Path`]'s part of the payment
+       ///
+       /// [`BlindedPath`]: crate::blinded_path::BlindedPath
        pub fee_msat: u64,
-       /// The CLTV delta added for this hop. For the last hop, this should be the full CLTV value
-       /// expected at the destination, in excess of the current block height.
+       /// The CLTV delta added for this hop.
+       /// If this is the last hop in [`Path::hops`]:
+       /// * if we're sending to a [`BlindedPath`], this is the CLTV delta for the entire blinded path
+       /// * otherwise, this is the CLTV delta expected at the destination
+       ///
+       /// [`BlindedPath`]: crate::blinded_path::BlindedPath
        pub cltv_expiry_delta: u32,
 }
 
@@ -241,16 +256,82 @@ impl_writeable_tlv_based!(RouteHop, {
        (10, cltv_expiry_delta, required),
 });
 
+/// The blinded portion of a [`Path`], if we're routing to a recipient who provided blinded paths in
+/// their BOLT12 [`Invoice`].
+///
+/// [`Invoice`]: crate::offers::invoice::Invoice
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct BlindedTail {
+       /// The hops of the [`BlindedPath`] provided by the recipient.
+       ///
+       /// [`BlindedPath`]: crate::blinded_path::BlindedPath
+       pub hops: Vec<BlindedHop>,
+       /// The blinding point of the [`BlindedPath`] provided by the recipient.
+       ///
+       /// [`BlindedPath`]: crate::blinded_path::BlindedPath
+       pub blinding_point: PublicKey,
+       /// Excess CLTV delta added to the recipient's CLTV expiry to deter intermediate nodes from
+       /// inferring the destination. May be 0.
+       pub excess_final_cltv_expiry_delta: u32,
+       /// The total amount paid on this [`Path`], excluding the fees.
+       pub final_value_msat: u64,
+}
+
+impl_writeable_tlv_based!(BlindedTail, {
+       (0, hops, vec_type),
+       (2, blinding_point, required),
+       (4, excess_final_cltv_expiry_delta, required),
+       (6, final_value_msat, required),
+});
+
+/// A path in a [`Route`] to the payment recipient. Must always be at least length one.
+/// If no [`Path::blinded_tail`] is present, then [`Path::hops`] length may be up to 19.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub struct Path {
+       /// The list of unblinded hops in this [`Path`]. Must be at least length one.
+       pub hops: Vec<RouteHop>,
+       /// The blinded path at which this path terminates, if we're sending to one, and its metadata.
+       pub blinded_tail: Option<BlindedTail>,
+}
+
+impl Path {
+       /// Gets the fees for a given path, excluding any excess paid to the recipient.
+       pub fn fee_msat(&self) -> u64 {
+               match &self.blinded_tail {
+                       Some(_) => self.hops.iter().map(|hop| hop.fee_msat).sum::<u64>(),
+                       None => {
+                               // Do not count last hop of each path since that's the full value of the payment
+                               self.hops.split_last().map_or(0,
+                                       |(_, path_prefix)| path_prefix.iter().map(|hop| hop.fee_msat).sum())
+                       }
+               }
+       }
+
+       /// Gets the total amount paid on this [`Path`], excluding the fees.
+       pub fn final_value_msat(&self) -> u64 {
+               match &self.blinded_tail {
+                       Some(blinded_tail) => blinded_tail.final_value_msat,
+                       None => self.hops.last().map_or(0, |hop| hop.fee_msat)
+               }
+       }
+
+       /// Gets the final hop's CLTV expiry delta.
+       pub fn final_cltv_expiry_delta(&self) -> Option<u32> {
+               match &self.blinded_tail {
+                       Some(_) => None,
+                       None => self.hops.last().map(|hop| hop.cltv_expiry_delta)
+               }
+       }
+}
+
 /// A route directs a payment from the sender (us) to the recipient. If the recipient supports MPP,
 /// it can take multiple paths. Each path is composed of one or more hops through the network.
 #[derive(Clone, Hash, PartialEq, Eq)]
 pub struct Route {
-       /// The list of routes taken for a single (potentially-)multi-part payment. The pubkey of the
-       /// last RouteHop in each path must be the same. Each entry represents a list of hops, NOT
-       /// INCLUDING our own, where the last hop is the destination. Thus, this must always be at
-       /// least length one. While the maximum length of any given path is variable, keeping the length
-       /// of any path less or equal to 19 should currently ensure it is viable.
-       pub paths: Vec<Vec<RouteHop>>,
+       /// The list of [`Path`]s taken for a single (potentially-)multi-part payment. If no
+       /// [`BlindedTail`]s are present, then the pubkey of the last [`RouteHop`] in each path must be
+       /// the same.
+       pub paths: Vec<Path>,
        /// The `payment_params` parameter passed to [`find_route`].
        /// This is used by `ChannelManager` to track information which may be required for retries,
        /// provided back to you via [`Event::PaymentPathFailed`].
@@ -259,33 +340,19 @@ pub struct Route {
        pub payment_params: Option<PaymentParameters>,
 }
 
-pub(crate) trait RoutePath {
-       /// Gets the fees for a given path, excluding any excess paid to the recipient.
-       fn get_path_fees(&self) -> u64;
-}
-impl RoutePath for Vec<RouteHop> {
-       fn get_path_fees(&self) -> u64 {
-               // Do not count last hop of each path since that's the full value of the payment
-               self.split_last().map(|(_, path_prefix)| path_prefix).unwrap_or(&[])
-                       .iter().map(|hop| &hop.fee_msat)
-                       .sum()
-       }
-}
-
 impl Route {
        /// Returns the total amount of fees paid on this [`Route`].
        ///
        /// This doesn't include any extra payment made to the recipient, which can happen in excess of
        /// the amount passed to [`find_route`]'s `params.final_value_msat`.
        pub fn get_total_fees(&self) -> u64 {
-               self.paths.iter().map(|path| path.get_path_fees()).sum()
+               self.paths.iter().map(|path| path.fee_msat()).sum()
        }
 
-       /// Returns the total amount paid on this [`Route`], excluding the fees.
+       /// Returns the total amount paid on this [`Route`], excluding the fees. Might be more than
+       /// requested if we had to reach htlc_minimum_msat.
        pub fn get_total_amount(&self) -> u64 {
-               return self.paths.iter()
-                       .map(|path| path.split_last().map(|(hop, _)| hop.fee_msat).unwrap_or(0))
-                       .sum();
+               self.paths.iter().map(|path| path.final_value_msat()).sum()
        }
 }
 
@@ -296,14 +363,25 @@ impl Writeable for Route {
        fn write<W: crate::util::ser::Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
                write_ver_prefix!(writer, SERIALIZATION_VERSION, MIN_SERIALIZATION_VERSION);
                (self.paths.len() as u64).write(writer)?;
-               for hops in self.paths.iter() {
-                       (hops.len() as u8).write(writer)?;
-                       for hop in hops.iter() {
+               let mut blinded_tails = Vec::new();
+               for path in self.paths.iter() {
+                       (path.hops.len() as u8).write(writer)?;
+                       for (idx, hop) in path.hops.iter().enumerate() {
                                hop.write(writer)?;
+                               if let Some(blinded_tail) = &path.blinded_tail {
+                                       if blinded_tails.is_empty() {
+                                               blinded_tails = Vec::with_capacity(path.hops.len());
+                                               for _ in 0..idx {
+                                                       blinded_tails.push(None);
+                                               }
+                                       }
+                                       blinded_tails.push(Some(blinded_tail));
+                               } else if !blinded_tails.is_empty() { blinded_tails.push(None); }
                        }
                }
                write_tlv_fields!(writer, {
                        (1, self.payment_params, option),
+                       (2, blinded_tails, optional_vec),
                });
                Ok(())
        }
@@ -325,12 +403,19 @@ impl Readable for Route {
                        if hops.is_empty() { return Err(DecodeError::InvalidValue); }
                        min_final_cltv_expiry_delta =
                                cmp::min(min_final_cltv_expiry_delta, hops.last().unwrap().cltv_expiry_delta);
-                       paths.push(hops);
+                       paths.push(Path { hops, blinded_tail: None });
                }
-               let mut payment_params = None;
-               read_tlv_fields!(reader, {
+               _init_and_read_tlv_fields!(reader, {
                        (1, payment_params, (option: ReadableArgs, min_final_cltv_expiry_delta)),
+                       (2, blinded_tails, optional_vec),
                });
+               let blinded_tails = blinded_tails.unwrap_or(Vec::new());
+               if blinded_tails.len() != 0 {
+                       if blinded_tails.len() != paths.len() { return Err(DecodeError::InvalidValue) }
+                       for (mut path, blinded_tail_opt) in paths.iter_mut().zip(blinded_tails.into_iter()) {
+                               path.blinded_tail = blinded_tail_opt;
+                       }
+               }
                Ok(Route { paths, payment_params })
        }
 }
@@ -420,7 +505,7 @@ pub struct PaymentParameters {
        pub features: Option<InvoiceFeatures>,
 
        /// Hints for routing to the payee, containing channels connecting the payee to public nodes.
-       pub route_hints: Vec<RouteHint>,
+       pub route_hints: Hints,
 
        /// Expiration of a payment to the payee, in seconds relative to the UNIX epoch.
        pub expiry_time: Option<u64>,
@@ -459,15 +544,22 @@ pub struct PaymentParameters {
 
 impl Writeable for PaymentParameters {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               let mut clear_hints = &vec![];
+               let mut blinded_hints = &vec![];
+               match &self.route_hints {
+                       Hints::Clear(hints) => clear_hints = hints,
+                       Hints::Blinded(hints) => blinded_hints = hints,
+               }
                write_tlv_fields!(writer, {
                        (0, self.payee_pubkey, required),
                        (1, self.max_total_cltv_expiry_delta, required),
                        (2, self.features, option),
                        (3, self.max_path_count, required),
-                       (4, self.route_hints, vec_type),
+                       (4, *clear_hints, vec_type),
                        (5, self.max_channel_saturation_power_of_half, required),
                        (6, self.expiry_time, option),
                        (7, self.previously_failed_channels, vec_type),
+                       (8, *blinded_hints, optional_vec),
                        (9, self.final_cltv_expiry_delta, required),
                });
                Ok(())
@@ -485,14 +577,23 @@ impl ReadableArgs<u32> for PaymentParameters {
                        (5, max_channel_saturation_power_of_half, (default_value, 2)),
                        (6, expiry_time, option),
                        (7, previously_failed_channels, vec_type),
+                       (8, blinded_route_hints, optional_vec),
                        (9, final_cltv_expiry_delta, (default_value, default_final_cltv_expiry_delta)),
                });
+               let clear_route_hints = route_hints.unwrap_or(vec![]);
+               let blinded_route_hints = blinded_route_hints.unwrap_or(vec![]);
+               let route_hints = if blinded_route_hints.len() != 0 {
+                       if clear_route_hints.len() != 0 { return Err(DecodeError::InvalidValue) }
+                       Hints::Blinded(blinded_route_hints)
+               } else {
+                       Hints::Clear(clear_route_hints)
+               };
                Ok(Self {
                        payee_pubkey: _init_tlv_based_struct_field!(payee_pubkey, required),
                        max_total_cltv_expiry_delta: _init_tlv_based_struct_field!(max_total_cltv_expiry_delta, (default_value, unused)),
                        features,
                        max_path_count: _init_tlv_based_struct_field!(max_path_count, (default_value, unused)),
-                       route_hints: route_hints.unwrap_or(Vec::new()),
+                       route_hints,
                        max_channel_saturation_power_of_half: _init_tlv_based_struct_field!(max_channel_saturation_power_of_half, (default_value, unused)),
                        expiry_time,
                        previously_failed_channels: previously_failed_channels.unwrap_or(Vec::new()),
@@ -511,7 +612,7 @@ impl PaymentParameters {
                Self {
                        payee_pubkey,
                        features: None,
-                       route_hints: vec![],
+                       route_hints: Hints::Clear(vec![]),
                        expiry_time: None,
                        max_total_cltv_expiry_delta: DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA,
                        max_path_count: DEFAULT_MAX_PATH_COUNT,
@@ -540,7 +641,7 @@ impl PaymentParameters {
        ///
        /// This is not exported to bindings users since bindings don't support move semantics
        pub fn with_route_hints(self, route_hints: Vec<RouteHint>) -> Self {
-               Self { route_hints, ..self }
+               Self { route_hints: Hints::Clear(route_hints), ..self }
        }
 
        /// Includes a payment expiration in seconds relative to the UNIX epoch.
@@ -572,6 +673,16 @@ impl PaymentParameters {
        }
 }
 
+/// Routing hints for the tail of the route.
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+pub enum Hints {
+       /// The recipient provided blinded paths and payinfo to reach them. The blinded paths themselves
+       /// will be included in the final [`Route`].
+       Blinded(Vec<(BlindedPayInfo, BlindedPath)>),
+       /// The recipient included these route hints in their BOLT11 invoice.
+       Clear(Vec<RouteHint>),
+}
+
 /// A list of hops along a payment path terminating with a channel to the recipient.
 #[derive(Clone, Debug, Hash, Eq, PartialEq)]
 pub struct RouteHint(pub Vec<RouteHintHop>);
@@ -1021,12 +1132,18 @@ where L::Target: Logger {
                return Err(LightningError{err: "Cannot send a payment of 0 msat".to_owned(), action: ErrorAction::IgnoreError});
        }
 
-       for route in payment_params.route_hints.iter() {
-               for hop in &route.0 {
-                       if hop.src_node_id == payment_params.payee_pubkey {
-                               return Err(LightningError{err: "Route hint cannot have the payee as the source.".to_owned(), action: ErrorAction::IgnoreError});
+       match &payment_params.route_hints {
+               Hints::Clear(hints) => {
+                       for route in hints.iter() {
+                               for hop in &route.0 {
+                                       if hop.src_node_id == payment_params.payee_pubkey {
+                                               return Err(LightningError{err: "Route hint cannot have the payee as the source.".to_owned(), action: ErrorAction::IgnoreError});
+                                       }
+                               }
                        }
-               }
+               },
+               _ => return Err(LightningError{err: "Routing to blinded paths isn't supported yet".to_owned(), action: ErrorAction::IgnoreError}),
+
        }
        if payment_params.max_total_cltv_expiry_delta <= final_cltv_expiry_delta {
                return Err(LightningError{err: "Can't find a route where the maximum total CLTV expiry delta is below the final CLTV expiry.".to_owned(), action: ErrorAction::IgnoreError});
@@ -1551,7 +1668,11 @@ where L::Target: Logger {
                // If a caller provided us with last hops, add them to routing targets. Since this happens
                // earlier than general path finding, they will be somewhat prioritized, although currently
                // it matters only if the fees are exactly the same.
-               for route in payment_params.route_hints.iter().filter(|route| !route.0.is_empty()) {
+               let route_hints = match &payment_params.route_hints {
+                       Hints::Clear(hints) => hints,
+                       _ => return Err(LightningError{err: "Routing to blinded paths isn't supported yet".to_owned(), action: ErrorAction::IgnoreError}),
+               };
+               for route in route_hints.iter().filter(|route| !route.0.is_empty()) {
                        let first_hop_in_route = &(route.0)[0];
                        let have_hop_src_in_graph =
                                // Only add the hops in this route to our candidate set if either
@@ -1966,8 +2087,14 @@ where L::Target: Logger {
                }
        }
 
+       let mut paths: Vec<Path> = Vec::new();
+       for results_vec in selected_paths {
+               let mut hops = Vec::with_capacity(results_vec.len());
+               for res in results_vec { hops.push(res?); }
+               paths.push(Path { hops, blinded_tail: None });
+       }
        let route = Route {
-               paths: selected_paths.into_iter().map(|path| path.into_iter().collect()).collect::<Result<Vec<_>, _>>()?,
+               paths,
                payment_params: Some(payment_params.clone()),
        };
        log_info!(logger, "Got route to {}: {}", payment_params.payee_pubkey, log_route!(route));
@@ -1989,14 +2116,14 @@ fn add_random_cltv_offset(route: &mut Route, payment_params: &PaymentParameters,
 
                // Remember the last three nodes of the random walk and avoid looping back on them.
                // Init with the last three nodes from the actual path, if possible.
-               let mut nodes_to_avoid: [NodeId; 3] = [NodeId::from_pubkey(&path.last().unwrap().pubkey),
-                       NodeId::from_pubkey(&path.get(path.len().saturating_sub(2)).unwrap().pubkey),
-                       NodeId::from_pubkey(&path.get(path.len().saturating_sub(3)).unwrap().pubkey)];
+               let mut nodes_to_avoid: [NodeId; 3] = [NodeId::from_pubkey(&path.hops.last().unwrap().pubkey),
+                       NodeId::from_pubkey(&path.hops.get(path.hops.len().saturating_sub(2)).unwrap().pubkey),
+                       NodeId::from_pubkey(&path.hops.get(path.hops.len().saturating_sub(3)).unwrap().pubkey)];
 
                // Choose the last publicly known node as the starting point for the random walk.
                let mut cur_hop: Option<NodeId> = None;
                let mut path_nonce = [0u8; 12];
-               if let Some(starting_hop) = path.iter().rev()
+               if let Some(starting_hop) = path.hops.iter().rev()
                        .find(|h| network_nodes.contains_key(&NodeId::from_pubkey(&h.pubkey))) {
                                cur_hop = Some(NodeId::from_pubkey(&starting_hop.pubkey));
                                path_nonce.copy_from_slice(&cur_hop.unwrap().as_slice()[..12]);
@@ -2045,7 +2172,7 @@ fn add_random_cltv_offset(route: &mut Route, payment_params: &PaymentParameters,
 
                // Limit the offset so we never exceed the max_total_cltv_expiry_delta. To improve plausibility,
                // we choose the limit to be the largest possible multiple of MEDIAN_HOP_CLTV_EXPIRY_DELTA.
-               let path_total_cltv_expiry_delta: u32 = path.iter().map(|h| h.cltv_expiry_delta).sum();
+               let path_total_cltv_expiry_delta: u32 = path.hops.iter().map(|h| h.cltv_expiry_delta).sum();
                let mut max_path_offset = payment_params.max_total_cltv_expiry_delta - path_total_cltv_expiry_delta;
                max_path_offset = cmp::max(
                        max_path_offset - (max_path_offset % MEDIAN_HOP_CLTV_EXPIRY_DELTA),
@@ -2053,7 +2180,11 @@ fn add_random_cltv_offset(route: &mut Route, payment_params: &PaymentParameters,
                shadow_ctlv_expiry_delta_offset = cmp::min(shadow_ctlv_expiry_delta_offset, max_path_offset);
 
                // Add 'shadow' CLTV offset to the final hop
-               if let Some(last_hop) = path.last_mut() {
+               if let Some(tail) = path.blinded_tail.as_mut() {
+                       tail.excess_final_cltv_expiry_delta = tail.excess_final_cltv_expiry_delta
+                               .checked_add(shadow_ctlv_expiry_delta_offset).unwrap_or(tail.excess_final_cltv_expiry_delta);
+               }
+               if let Some(last_hop) = path.hops.last_mut() {
                        last_hop.cltv_expiry_delta = last_hop.cltv_expiry_delta
                                .checked_add(shadow_ctlv_expiry_delta_offset).unwrap_or(last_hop.cltv_expiry_delta);
                }
@@ -2107,13 +2238,13 @@ fn build_route_from_hops_internal<L: Deref>(
                        u64::max_value()
                }
 
-               fn payment_path_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
+               fn payment_path_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
 
-               fn payment_path_successful(&mut self, _path: &[&RouteHop]) {}
+               fn payment_path_successful(&mut self, _path: &Path) {}
 
-               fn probe_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
+               fn probe_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
 
-               fn probe_successful(&mut self, _path: &[&RouteHop]) {}
+               fn probe_successful(&mut self, _path: &Path) {}
        }
 
        impl<'a> Writeable for HopScorer {
@@ -2141,10 +2272,11 @@ fn build_route_from_hops_internal<L: Deref>(
 
 #[cfg(test)]
 mod tests {
+       use crate::blinded_path::{BlindedHop, BlindedPath};
        use crate::routing::gossip::{NetworkGraph, P2PGossipSync, NodeId, EffectiveCapacity};
        use crate::routing::utxo::UtxoResult;
        use crate::routing::router::{get_route, build_route_from_hops_internal, add_random_cltv_offset, default_node_features,
-               PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RoutingFees,
+               BlindedTail, InFlightHtlcs, Path, PaymentParameters, Route, RouteHint, RouteHintHop, RouteHop, RoutingFees,
                DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA, MAX_PATH_LENGTH_ESTIMATE};
        use crate::routing::scoring::{ChannelUsage, FixedPenaltyScorer, Score, ProbabilisticScorer, ProbabilisticScoringParameters};
        use crate::routing::test_utils::{add_channel, add_or_update_node, build_graph, build_line_graph, id_to_feature_flags, get_nodes, update_channel};
@@ -2156,8 +2288,9 @@ mod tests {
        use crate::util::config::UserConfig;
        use crate::util::test_utils as ln_test_utils;
        use crate::util::chacha20::ChaCha20;
+       use crate::util::ser::{Readable, Writeable};
        #[cfg(c_bindings)]
-       use crate::util::ser::{Writeable, Writer};
+       use crate::util::ser::Writer;
 
        use bitcoin::hashes::Hash;
        use bitcoin::network::constants::Network;
@@ -2171,6 +2304,7 @@ mod tests {
        use bitcoin::secp256k1::{PublicKey,SecretKey};
        use bitcoin::secp256k1::Secp256k1;
 
+       use crate::io::Cursor;
        use crate::prelude::*;
        use crate::sync::Arc;
 
@@ -2228,21 +2362,21 @@ mod tests {
                } else { panic!(); }
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
 
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 100);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
 
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
        }
 
        #[test]
@@ -2264,7 +2398,7 @@ mod tests {
                } else { panic!(); }
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
        }
 
        #[test]
@@ -2391,7 +2525,7 @@ mod tests {
 
                // A payment above the minimum should pass
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 199_999_999, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
        }
 
        #[test]
@@ -2474,7 +2608,7 @@ mod tests {
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 60_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                // Overpay fees to hit htlc_minimum_msat.
-               let overpaid_fees = route.paths[0][0].fee_msat + route.paths[1][0].fee_msat;
+               let overpaid_fees = route.paths[0].hops[0].fee_msat + route.paths[1].hops[0].fee_msat;
                // TODO: this could be better balanced to overpay 10k and not 15k.
                assert_eq!(overpaid_fees, 15_000);
 
@@ -2520,16 +2654,16 @@ mod tests {
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 60_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                // Fine to overpay for htlc_minimum_msat if it allows us to save fee.
                assert_eq!(route.paths.len(), 1);
-               assert_eq!(route.paths[0][0].short_channel_id, 12);
-               let fees = route.paths[0][0].fee_msat;
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 12);
+               let fees = route.paths[0].hops[0].fee_msat;
                assert_eq!(fees, 5_000);
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 50_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                // Not fine to overpay for htlc_minimum_msat if it requires paying more than fee on
                // the other channel.
                assert_eq!(route.paths.len(), 1);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               let fees = route.paths[0][0].fee_msat;
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               let fees = route.paths[0].hops[0].fee_msat;
                assert_eq!(fees, 5_000);
        }
 
@@ -2576,21 +2710,21 @@ mod tests {
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::<Vec<_>>()), 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
 
-               assert_eq!(route.paths[0][0].pubkey, nodes[7]);
-               assert_eq!(route.paths[0][0].short_channel_id, 42);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &vec![0b11]); // it should also override our view of their features
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 42);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (13 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &vec![0b11]); // it should also override our view of their features
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
 
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 13);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 13);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(13));
        }
 
        #[test]
@@ -2617,21 +2751,21 @@ mod tests {
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::<Vec<_>>()), 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[7]);
-               assert_eq!(route.paths[0][0].short_channel_id, 42);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &vec![0b11]); // it should also override our view of their features
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 13);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+               assert_eq!(route.paths[0].hops.len(), 2);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 42);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (13 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &vec![0b11]); // it should also override our view of their features
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 13);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(13));
 
                // Note that we don't test disabling node 3 and failing to route to it, as we (somewhat
                // naively) assume that the user checked the feature bits on the invoice, which override
@@ -2649,48 +2783,48 @@ mod tests {
                // Route to 1 via 2 and 3 because our channel to 1 is disabled
                let payment_params = PaymentParameters::from_node_id(nodes[0], 42);
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 3);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (3 << 4) | 2);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[0]);
-               assert_eq!(route.paths[0][2].short_channel_id, 3);
-               assert_eq!(route.paths[0][2].fee_msat, 100);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(1));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops.len(), 3);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (3 << 4) | 2);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[0]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 3);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(1));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(3));
 
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let payment_params = PaymentParameters::from_node_id(nodes[2], 42);
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::<Vec<_>>()), 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
 
-               assert_eq!(route.paths[0][0].pubkey, nodes[7]);
-               assert_eq!(route.paths[0][0].short_channel_id, 42);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &vec![0b11]);
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 42);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (13 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &vec![0b11]);
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
 
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 13);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 13);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(13));
        }
 
        fn last_hops(nodes: &Vec<PublicKey>) -> Vec<RouteHint> {
@@ -2805,44 +2939,44 @@ mod tests {
 
                let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops_multi_private_channels(&nodes));
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 5);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 100);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 0);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (6 << 4) | 1);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[4]);
-               assert_eq!(route.paths[0][2].short_channel_id, 6);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, (11 << 4) | 1);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(5));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(6));
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][3].short_channel_id, 11);
-               assert_eq!(route.paths[0][3].fee_msat, 0);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops.len(), 5);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (6 << 4) | 1);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[4]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 6);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, (11 << 4) | 1);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(5));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(6));
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, 11);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, (8 << 4) | 1);
                // If we have a peer in the node map, we'll use their features here since we don't have
                // a way of figuring out their features from the invoice:
-               assert_eq!(route.paths[0][3].node_features.le_flags(), &id_to_feature_flags(4));
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &id_to_feature_flags(11));
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &id_to_feature_flags(11));
 
-               assert_eq!(route.paths[0][4].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][4].short_channel_id, 8);
-               assert_eq!(route.paths[0][4].fee_msat, 100);
-               assert_eq!(route.paths[0][4].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[4].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[4].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[4].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[4].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        fn empty_last_hop(nodes: &Vec<PublicKey>) -> Vec<RouteHint> {
@@ -2881,44 +3015,44 @@ mod tests {
                // Test handling of an empty RouteHint passed in Invoice.
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 5);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 100);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 0);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (6 << 4) | 1);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[4]);
-               assert_eq!(route.paths[0][2].short_channel_id, 6);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, (11 << 4) | 1);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(5));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(6));
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][3].short_channel_id, 11);
-               assert_eq!(route.paths[0][3].fee_msat, 0);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops.len(), 5);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (6 << 4) | 1);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[4]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 6);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, (11 << 4) | 1);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(5));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(6));
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, 11);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, (8 << 4) | 1);
                // If we have a peer in the node map, we'll use their features here since we don't have
                // a way of figuring out their features from the invoice:
-               assert_eq!(route.paths[0][3].node_features.le_flags(), &id_to_feature_flags(4));
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &id_to_feature_flags(11));
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &id_to_feature_flags(11));
 
-               assert_eq!(route.paths[0][4].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][4].short_channel_id, 8);
-               assert_eq!(route.paths[0][4].fee_msat, 100);
-               assert_eq!(route.paths[0][4].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[4].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[4].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[4].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[4].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        /// Builds a trivial last-hop hint that passes through the two nodes given, with channel 0xff00
@@ -2987,35 +3121,35 @@ mod tests {
                });
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 4);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, 65);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 81);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][2].short_channel_id, last_hops[0].0[0].short_channel_id);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, 129);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(4));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][3].short_channel_id, last_hops[0].0[1].short_channel_id);
-               assert_eq!(route.paths[0][3].fee_msat, 100);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops.len(), 4);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, 65);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 81);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, last_hops[0].0[0].short_channel_id);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, 129);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, last_hops[0].0[1].short_channel_id);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        #[test]
@@ -3059,35 +3193,35 @@ mod tests {
                });
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &[42u8; 32]).unwrap();
-               assert_eq!(route.paths[0].len(), 4);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 200);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, 65);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 81);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, non_announced_pubkey);
-               assert_eq!(route.paths[0][2].short_channel_id, last_hops[0].0[0].short_channel_id);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, 129);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][3].short_channel_id, last_hops[0].0[1].short_channel_id);
-               assert_eq!(route.paths[0][3].fee_msat, 100);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops.len(), 4);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, 65);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 81);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, non_announced_pubkey);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, last_hops[0].0[0].short_channel_id);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, 129);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, last_hops[0].0[1].short_channel_id);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        fn last_hops_with_public_channel(nodes: &Vec<PublicKey>) -> Vec<RouteHint> {
@@ -3141,44 +3275,44 @@ mod tests {
                // which would be handled in the same manner.
 
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 5);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 100);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 0);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (6 << 4) | 1);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[4]);
-               assert_eq!(route.paths[0][2].short_channel_id, 6);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, (11 << 4) | 1);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(5));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(6));
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][3].short_channel_id, 11);
-               assert_eq!(route.paths[0][3].fee_msat, 0);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops.len(), 5);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (6 << 4) | 1);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[4]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 6);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, (11 << 4) | 1);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(5));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(6));
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, 11);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, (8 << 4) | 1);
                // If we have a peer in the node map, we'll use their features here since we don't have
                // a way of figuring out their features from the invoice:
-               assert_eq!(route.paths[0][3].node_features.le_flags(), &id_to_feature_flags(4));
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &id_to_feature_flags(11));
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &id_to_feature_flags(11));
 
-               assert_eq!(route.paths[0][4].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][4].short_channel_id, 8);
-               assert_eq!(route.paths[0][4].fee_msat, 100);
-               assert_eq!(route.paths[0][4].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[4].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[4].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[4].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[4].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        #[test]
@@ -3194,99 +3328,99 @@ mod tests {
                let mut last_hops = last_hops(&nodes);
                let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops.clone());
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::<Vec<_>>()), 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 2);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][0].short_channel_id, 42);
-               assert_eq!(route.paths[0][0].fee_msat, 0);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (8 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &vec![0b11]);
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][1].short_channel_id, 8);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops.len(), 2);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 42);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &vec![0b11]);
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &Vec::<u8>::new()); // No feature flags will meet the relevant-to-channel conversion
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
 
                last_hops[0].0[0].fees.base_msat = 1000;
 
                // Revert to via 6 as the fee on 8 goes up
                let payment_params = PaymentParameters::from_node_id(nodes[6], 42).with_route_hints(last_hops);
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 4);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 200); // fee increased as its % of value transferred across node
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 100);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (7 << 4) | 1);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[5]);
-               assert_eq!(route.paths[0][2].short_channel_id, 7);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, (10 << 4) | 1);
+               assert_eq!(route.paths[0].hops.len(), 4);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 200); // fee increased as its % of value transferred across node
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (7 << 4) | 1);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[5]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 7);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, (10 << 4) | 1);
                // If we have a peer in the node map, we'll use their features here since we don't have
                // a way of figuring out their features from the invoice:
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(6));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(7));
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(6));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(7));
 
-               assert_eq!(route.paths[0][3].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][3].short_channel_id, 10);
-               assert_eq!(route.paths[0][3].fee_msat, 100);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, 10);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 100);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
 
                // ...but still use 8 for larger payments as 6 has a variable feerate
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 2000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               assert_eq!(route.paths[0].len(), 5);
-
-               assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-               assert_eq!(route.paths[0][0].short_channel_id, 2);
-               assert_eq!(route.paths[0][0].fee_msat, 3000);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (4 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(2));
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(2));
-
-               assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-               assert_eq!(route.paths[0][1].short_channel_id, 4);
-               assert_eq!(route.paths[0][1].fee_msat, 0);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, (6 << 4) | 1);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(4));
-
-               assert_eq!(route.paths[0][2].pubkey, nodes[4]);
-               assert_eq!(route.paths[0][2].short_channel_id, 6);
-               assert_eq!(route.paths[0][2].fee_msat, 0);
-               assert_eq!(route.paths[0][2].cltv_expiry_delta, (11 << 4) | 1);
-               assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(5));
-               assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(6));
-
-               assert_eq!(route.paths[0][3].pubkey, nodes[3]);
-               assert_eq!(route.paths[0][3].short_channel_id, 11);
-               assert_eq!(route.paths[0][3].fee_msat, 1000);
-               assert_eq!(route.paths[0][3].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops.len(), 5);
+
+               assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 3000);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (4 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(2));
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(2));
+
+               assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 4);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (6 << 4) | 1);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(4));
+
+               assert_eq!(route.paths[0].hops[2].pubkey, nodes[4]);
+               assert_eq!(route.paths[0].hops[2].short_channel_id, 6);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, (11 << 4) | 1);
+               assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(5));
+               assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(6));
+
+               assert_eq!(route.paths[0].hops[3].pubkey, nodes[3]);
+               assert_eq!(route.paths[0].hops[3].short_channel_id, 11);
+               assert_eq!(route.paths[0].hops[3].fee_msat, 1000);
+               assert_eq!(route.paths[0].hops[3].cltv_expiry_delta, (8 << 4) | 1);
                // If we have a peer in the node map, we'll use their features here since we don't have
                // a way of figuring out their features from the invoice:
-               assert_eq!(route.paths[0][3].node_features.le_flags(), &id_to_feature_flags(4));
-               assert_eq!(route.paths[0][3].channel_features.le_flags(), &id_to_feature_flags(11));
+               assert_eq!(route.paths[0].hops[3].node_features.le_flags(), &id_to_feature_flags(4));
+               assert_eq!(route.paths[0].hops[3].channel_features.le_flags(), &id_to_feature_flags(11));
 
-               assert_eq!(route.paths[0][4].pubkey, nodes[6]);
-               assert_eq!(route.paths[0][4].short_channel_id, 8);
-               assert_eq!(route.paths[0][4].fee_msat, 2000);
-               assert_eq!(route.paths[0][4].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[4].pubkey, nodes[6]);
+               assert_eq!(route.paths[0].hops[4].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[4].fee_msat, 2000);
+               assert_eq!(route.paths[0].hops[4].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[4].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[4].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
        }
 
        fn do_unannounced_path_test(last_hop_htlc_max: Option<u64>, last_hop_fee_prop: u32, outbound_capacity_msat: u64, route_val: u64) -> Result<Route, LightningError> {
@@ -3327,21 +3461,21 @@ mod tests {
 
                let middle_node_id = PublicKey::from_secret_key(&Secp256k1::new(), &SecretKey::from_slice(&hex::decode(format!("{:02}", 42).repeat(32)).unwrap()[..]).unwrap());
                let target_node_id = PublicKey::from_secret_key(&Secp256k1::new(), &SecretKey::from_slice(&hex::decode(format!("{:02}", 43).repeat(32)).unwrap()[..]).unwrap());
-               assert_eq!(route.paths[0].len(), 2);
+               assert_eq!(route.paths[0].hops.len(), 2);
 
-               assert_eq!(route.paths[0][0].pubkey, middle_node_id);
-               assert_eq!(route.paths[0][0].short_channel_id, 42);
-               assert_eq!(route.paths[0][0].fee_msat, 1001);
-               assert_eq!(route.paths[0][0].cltv_expiry_delta, (8 << 4) | 1);
-               assert_eq!(route.paths[0][0].node_features.le_flags(), &[0b11]);
-               assert_eq!(route.paths[0][0].channel_features.le_flags(), &[0; 0]); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[0].pubkey, middle_node_id);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 42);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 1001);
+               assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (8 << 4) | 1);
+               assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &[0b11]);
+               assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &[0; 0]); // We can't learn any flags from invoices, sadly
 
-               assert_eq!(route.paths[0][1].pubkey, target_node_id);
-               assert_eq!(route.paths[0][1].short_channel_id, 8);
-               assert_eq!(route.paths[0][1].fee_msat, 1000000);
-               assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-               assert_eq!(route.paths[0][1].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
-               assert_eq!(route.paths[0][1].channel_features.le_flags(), &[0; 0]); // We can't learn any flags from invoices, sadly
+               assert_eq!(route.paths[0].hops[1].pubkey, target_node_id);
+               assert_eq!(route.paths[0].hops[1].short_channel_id, 8);
+               assert_eq!(route.paths[0].hops[1].fee_msat, 1000000);
+               assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+               assert_eq!(route.paths[0].hops[1].node_features.le_flags(), default_node_features().le_flags()); // We dont pass flags in from invoices yet
+               assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &[0; 0]); // We can't learn any flags from invoices, sadly
        }
 
        #[test]
@@ -3446,9 +3580,9 @@ mod tests {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 250_000_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
-                       assert_eq!(path.len(), 2);
-                       assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                       assert_eq!(path.last().unwrap().fee_msat, 250_000_000);
+                       assert_eq!(path.hops.len(), 2);
+                       assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                       assert_eq!(path.final_value_msat(), 250_000_000);
                }
 
                // Check that setting next_outbound_htlc_limit_msat in first_hops limits the channels.
@@ -3482,9 +3616,9 @@ mod tests {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&our_chans.iter().collect::<Vec<_>>()), 200_000_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
-                       assert_eq!(path.len(), 2);
-                       assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                       assert_eq!(path.last().unwrap().fee_msat, 200_000_000);
+                       assert_eq!(path.hops.len(), 2);
+                       assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                       assert_eq!(path.final_value_msat(), 200_000_000);
                }
 
                // Enable channel #1 back.
@@ -3529,9 +3663,9 @@ mod tests {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 15_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
-                       assert_eq!(path.len(), 2);
-                       assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                       assert_eq!(path.last().unwrap().fee_msat, 15_000);
+                       assert_eq!(path.hops.len(), 2);
+                       assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                       assert_eq!(path.final_value_msat(), 15_000);
                }
 
                // Now let's see if routing works if we know only capacity from the UTXO.
@@ -3600,9 +3734,9 @@ mod tests {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 15_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
-                       assert_eq!(path.len(), 2);
-                       assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                       assert_eq!(path.last().unwrap().fee_msat, 15_000);
+                       assert_eq!(path.hops.len(), 2);
+                       assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                       assert_eq!(path.final_value_msat(), 15_000);
                }
 
                // Now let's see if routing chooses htlc_maximum_msat over UTXO capacity.
@@ -3632,9 +3766,9 @@ mod tests {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 10_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
-                       assert_eq!(path.len(), 2);
-                       assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                       assert_eq!(path.last().unwrap().fee_msat, 10_000);
+                       assert_eq!(path.hops.len(), 2);
+                       assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                       assert_eq!(path.final_value_msat(), 10_000);
                }
        }
 
@@ -3745,9 +3879,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 4);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[3]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 4);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[3]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 49_000);
                }
@@ -3758,9 +3892,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 4);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[3]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 4);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[3]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 50_000);
                }
@@ -3806,9 +3940,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 2);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 2);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 50_000);
                }
@@ -3952,9 +4086,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 2);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 2);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 250_000);
                }
@@ -3966,9 +4100,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 2);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 2);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 290_000);
                }
@@ -4131,8 +4265,8 @@ mod tests {
 
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.last().unwrap().pubkey, nodes[3]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[3]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 300_000);
                }
@@ -4293,9 +4427,9 @@ mod tests {
                        let mut total_value_transferred_msat = 0;
                        let mut total_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.last().unwrap().pubkey, nodes[3]);
-                               total_value_transferred_msat += path.last().unwrap().fee_msat;
-                               for hop in path {
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[3]);
+                               total_value_transferred_msat += path.final_value_msat();
+                               for hop in &path.hops {
                                        total_paid_msat += hop.fee_msat;
                                }
                        }
@@ -4470,8 +4604,8 @@ mod tests {
 
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.last().unwrap().pubkey, nodes[3]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[3]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 200_000);
                        assert_eq!(route.get_total_fees(), 150_000);
@@ -4567,16 +4701,16 @@ mod tests {
                // overpay at all.
                let mut route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                assert_eq!(route.paths.len(), 2);
-               route.paths.sort_by_key(|path| path[0].short_channel_id);
+               route.paths.sort_by_key(|path| path.hops[0].short_channel_id);
                // Paths are manually ordered ordered by SCID, so:
                // * the first is channel 1 (0 fee, but 99 sat maximum) -> channel 3 -> channel 42
                // * the second is channel 2 (1 msat fee) -> channel 4 -> channel 42
-               assert_eq!(route.paths[0][0].short_channel_id, 1);
-               assert_eq!(route.paths[0][0].fee_msat, 0);
-               assert_eq!(route.paths[0][2].fee_msat, 99_000);
-               assert_eq!(route.paths[1][0].short_channel_id, 2);
-               assert_eq!(route.paths[1][0].fee_msat, 1);
-               assert_eq!(route.paths[1][2].fee_msat, 1_000);
+               assert_eq!(route.paths[0].hops[0].short_channel_id, 1);
+               assert_eq!(route.paths[0].hops[0].fee_msat, 0);
+               assert_eq!(route.paths[0].hops[2].fee_msat, 99_000);
+               assert_eq!(route.paths[1].hops[0].short_channel_id, 2);
+               assert_eq!(route.paths[1].hops[0].fee_msat, 1);
+               assert_eq!(route.paths[1].hops[2].fee_msat, 1_000);
                assert_eq!(route.get_total_fees(), 1);
                assert_eq!(route.get_total_amount(), 100_000);
        }
@@ -4696,9 +4830,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 2);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 2);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 125_000);
                }
@@ -4709,9 +4843,9 @@ mod tests {
                        assert_eq!(route.paths.len(), 2);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
-                               assert_eq!(path.len(), 2);
-                               assert_eq!(path.last().unwrap().pubkey, nodes[2]);
-                               total_amount_paid_msat += path.last().unwrap().fee_msat;
+                               assert_eq!(path.hops.len(), 2);
+                               assert_eq!(path.hops.last().unwrap().pubkey, nodes[2]);
+                               total_amount_paid_msat += path.final_value_msat();
                        }
                        assert_eq!(total_amount_paid_msat, 90_000);
                }
@@ -4846,28 +4980,28 @@ mod tests {
                        // Now ensure the route flows simply over nodes 1 and 4 to 6.
                        let route = get_route(&our_id, &payment_params, &network.read_only(), None, 10_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
-                       assert_eq!(route.paths[0].len(), 3);
-
-                       assert_eq!(route.paths[0][0].pubkey, nodes[1]);
-                       assert_eq!(route.paths[0][0].short_channel_id, 6);
-                       assert_eq!(route.paths[0][0].fee_msat, 100);
-                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (5 << 4) | 0);
-                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(1));
-                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(6));
-
-                       assert_eq!(route.paths[0][1].pubkey, nodes[4]);
-                       assert_eq!(route.paths[0][1].short_channel_id, 5);
-                       assert_eq!(route.paths[0][1].fee_msat, 0);
-                       assert_eq!(route.paths[0][1].cltv_expiry_delta, (1 << 4) | 0);
-                       assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(4));
-                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(5));
-
-                       assert_eq!(route.paths[0][2].pubkey, nodes[6]);
-                       assert_eq!(route.paths[0][2].short_channel_id, 1);
-                       assert_eq!(route.paths[0][2].fee_msat, 10_000);
-                       assert_eq!(route.paths[0][2].cltv_expiry_delta, 42);
-                       assert_eq!(route.paths[0][2].node_features.le_flags(), &id_to_feature_flags(6));
-                       assert_eq!(route.paths[0][2].channel_features.le_flags(), &id_to_feature_flags(1));
+                       assert_eq!(route.paths[0].hops.len(), 3);
+
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[1]);
+                       assert_eq!(route.paths[0].hops[0].short_channel_id, 6);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 100);
+                       assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (5 << 4) | 0);
+                       assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(1));
+                       assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(6));
+
+                       assert_eq!(route.paths[0].hops[1].pubkey, nodes[4]);
+                       assert_eq!(route.paths[0].hops[1].short_channel_id, 5);
+                       assert_eq!(route.paths[0].hops[1].fee_msat, 0);
+                       assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, (1 << 4) | 0);
+                       assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(4));
+                       assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(5));
+
+                       assert_eq!(route.paths[0].hops[2].pubkey, nodes[6]);
+                       assert_eq!(route.paths[0].hops[2].short_channel_id, 1);
+                       assert_eq!(route.paths[0].hops[2].fee_msat, 10_000);
+                       assert_eq!(route.paths[0].hops[2].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0].hops[2].node_features.le_flags(), &id_to_feature_flags(6));
+                       assert_eq!(route.paths[0].hops[2].channel_features.le_flags(), &id_to_feature_flags(1));
                }
        }
 
@@ -4917,21 +5051,21 @@ mod tests {
                        // 200% fee charged channel 13 in the 1-to-2 direction.
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 90_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
-                       assert_eq!(route.paths[0].len(), 2);
-
-                       assert_eq!(route.paths[0][0].pubkey, nodes[7]);
-                       assert_eq!(route.paths[0][0].short_channel_id, 12);
-                       assert_eq!(route.paths[0][0].fee_msat, 90_000*2);
-                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 4) | 1);
-                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(8));
-                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(12));
-
-                       assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-                       assert_eq!(route.paths[0][1].short_channel_id, 13);
-                       assert_eq!(route.paths[0][1].fee_msat, 90_000);
-                       assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-                       assert_eq!(route.paths[0][1].node_features.le_flags(), &id_to_feature_flags(3));
-                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+                       assert_eq!(route.paths[0].hops.len(), 2);
+
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]);
+                       assert_eq!(route.paths[0].hops[0].short_channel_id, 12);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 90_000*2);
+                       assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (13 << 4) | 1);
+                       assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(8));
+                       assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(12));
+
+                       assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+                       assert_eq!(route.paths[0].hops[1].short_channel_id, 13);
+                       assert_eq!(route.paths[0].hops[1].fee_msat, 90_000);
+                       assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0].hops[1].node_features.le_flags(), &id_to_feature_flags(3));
+                       assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(13));
                }
        }
 
@@ -4983,21 +5117,21 @@ mod tests {
                        // expensive) channels 12-13 path.
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 90_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
-                       assert_eq!(route.paths[0].len(), 2);
-
-                       assert_eq!(route.paths[0][0].pubkey, nodes[7]);
-                       assert_eq!(route.paths[0][0].short_channel_id, 12);
-                       assert_eq!(route.paths[0][0].fee_msat, 90_000*2);
-                       assert_eq!(route.paths[0][0].cltv_expiry_delta, (13 << 4) | 1);
-                       assert_eq!(route.paths[0][0].node_features.le_flags(), &id_to_feature_flags(8));
-                       assert_eq!(route.paths[0][0].channel_features.le_flags(), &id_to_feature_flags(12));
-
-                       assert_eq!(route.paths[0][1].pubkey, nodes[2]);
-                       assert_eq!(route.paths[0][1].short_channel_id, 13);
-                       assert_eq!(route.paths[0][1].fee_msat, 90_000);
-                       assert_eq!(route.paths[0][1].cltv_expiry_delta, 42);
-                       assert_eq!(route.paths[0][1].node_features.le_flags(), channelmanager::provided_invoice_features(&config).le_flags());
-                       assert_eq!(route.paths[0][1].channel_features.le_flags(), &id_to_feature_flags(13));
+                       assert_eq!(route.paths[0].hops.len(), 2);
+
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[7]);
+                       assert_eq!(route.paths[0].hops[0].short_channel_id, 12);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 90_000*2);
+                       assert_eq!(route.paths[0].hops[0].cltv_expiry_delta, (13 << 4) | 1);
+                       assert_eq!(route.paths[0].hops[0].node_features.le_flags(), &id_to_feature_flags(8));
+                       assert_eq!(route.paths[0].hops[0].channel_features.le_flags(), &id_to_feature_flags(12));
+
+                       assert_eq!(route.paths[0].hops[1].pubkey, nodes[2]);
+                       assert_eq!(route.paths[0].hops[1].short_channel_id, 13);
+                       assert_eq!(route.paths[0].hops[1].fee_msat, 90_000);
+                       assert_eq!(route.paths[0].hops[1].cltv_expiry_delta, 42);
+                       assert_eq!(route.paths[0].hops[1].node_features.le_flags(), channelmanager::provided_invoice_features(&config).le_flags());
+                       assert_eq!(route.paths[0].hops[1].channel_features.le_flags(), &id_to_feature_flags(13));
                }
        }
 
@@ -5025,11 +5159,11 @@ mod tests {
                                &get_channel_details(Some(2), nodes[0], channelmanager::provided_init_features(&config), 10_000),
                        ]), 100_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
-                       assert_eq!(route.paths[0].len(), 1);
+                       assert_eq!(route.paths[0].hops.len(), 1);
 
-                       assert_eq!(route.paths[0][0].pubkey, nodes[0]);
-                       assert_eq!(route.paths[0][0].short_channel_id, 3);
-                       assert_eq!(route.paths[0][0].fee_msat, 100_000);
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[0]);
+                       assert_eq!(route.paths[0].hops[0].short_channel_id, 3);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 100_000);
                }
                {
                        let route = get_route(&our_id, &payment_params, &network_graph.read_only(), Some(&[
@@ -5037,17 +5171,17 @@ mod tests {
                                &get_channel_details(Some(2), nodes[0], channelmanager::provided_init_features(&config), 50_000),
                        ]), 100_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 2);
-                       assert_eq!(route.paths[0].len(), 1);
-                       assert_eq!(route.paths[1].len(), 1);
+                       assert_eq!(route.paths[0].hops.len(), 1);
+                       assert_eq!(route.paths[1].hops.len(), 1);
 
-                       assert!((route.paths[0][0].short_channel_id == 3 && route.paths[1][0].short_channel_id == 2) ||
-                               (route.paths[0][0].short_channel_id == 2 && route.paths[1][0].short_channel_id == 3));
+                       assert!((route.paths[0].hops[0].short_channel_id == 3 && route.paths[1].hops[0].short_channel_id == 2) ||
+                               (route.paths[0].hops[0].short_channel_id == 2 && route.paths[1].hops[0].short_channel_id == 3));
 
-                       assert_eq!(route.paths[0][0].pubkey, nodes[0]);
-                       assert_eq!(route.paths[0][0].fee_msat, 50_000);
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[0]);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 50_000);
 
-                       assert_eq!(route.paths[1][0].pubkey, nodes[0]);
-                       assert_eq!(route.paths[1][0].fee_msat, 50_000);
+                       assert_eq!(route.paths[1].hops[0].pubkey, nodes[0]);
+                       assert_eq!(route.paths[1].hops[0].fee_msat, 50_000);
                }
 
                {
@@ -5069,11 +5203,11 @@ mod tests {
                                &get_channel_details(Some(4), nodes[0], channelmanager::provided_init_features(&config), 1_000_000),
                        ]), 100_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                        assert_eq!(route.paths.len(), 1);
-                       assert_eq!(route.paths[0].len(), 1);
+                       assert_eq!(route.paths[0].hops.len(), 1);
 
-                       assert_eq!(route.paths[0][0].pubkey, nodes[0]);
-                       assert_eq!(route.paths[0][0].short_channel_id, 6);
-                       assert_eq!(route.paths[0][0].fee_msat, 100_000);
+                       assert_eq!(route.paths[0].hops[0].pubkey, nodes[0]);
+                       assert_eq!(route.paths[0].hops[0].short_channel_id, 6);
+                       assert_eq!(route.paths[0].hops[0].fee_msat, 100_000);
                }
        }
 
@@ -5091,7 +5225,7 @@ mod tests {
                        &our_id, &payment_params, &network_graph.read_only(), None, 100, 42,
                        Arc::clone(&logger), &scorer, &random_seed_bytes
                ).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
 
                assert_eq!(route.get_total_fees(), 100);
                assert_eq!(route.get_total_amount(), 100);
@@ -5104,7 +5238,7 @@ mod tests {
                        &our_id, &payment_params, &network_graph.read_only(), None, 100, 42,
                        Arc::clone(&logger), &scorer, &random_seed_bytes
                ).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
 
                assert_eq!(route.get_total_fees(), 300);
                assert_eq!(route.get_total_amount(), 100);
@@ -5124,10 +5258,10 @@ mod tests {
                        if short_channel_id == self.short_channel_id { u64::max_value() } else { 0 }
                }
 
-               fn payment_path_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
-               fn payment_path_successful(&mut self, _path: &[&RouteHop]) {}
-               fn probe_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
-               fn probe_successful(&mut self, _path: &[&RouteHop]) {}
+               fn payment_path_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
+               fn payment_path_successful(&mut self, _path: &Path) {}
+               fn probe_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
+               fn probe_successful(&mut self, _path: &Path) {}
        }
 
        struct BadNodeScorer {
@@ -5144,10 +5278,10 @@ mod tests {
                        if *target == self.node_id { u64::max_value() } else { 0 }
                }
 
-               fn payment_path_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
-               fn payment_path_successful(&mut self, _path: &[&RouteHop]) {}
-               fn probe_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
-               fn probe_successful(&mut self, _path: &[&RouteHop]) {}
+               fn payment_path_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
+               fn payment_path_successful(&mut self, _path: &Path) {}
+               fn probe_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
+               fn probe_successful(&mut self, _path: &Path) {}
        }
 
        #[test]
@@ -5165,7 +5299,7 @@ mod tests {
                        &our_id, &payment_params, &network_graph, None, 100, 42,
                        Arc::clone(&logger), &scorer, &random_seed_bytes
                ).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
 
                assert_eq!(route.get_total_fees(), 100);
                assert_eq!(route.get_total_amount(), 100);
@@ -5177,7 +5311,7 @@ mod tests {
                        &our_id, &payment_params, &network_graph, None, 100, 42,
                        Arc::clone(&logger), &scorer, &random_seed_bytes
                ).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
 
                assert_eq!(route.get_total_fees(), 300);
                assert_eq!(route.get_total_amount(), 100);
@@ -5199,7 +5333,7 @@ mod tests {
        #[test]
        fn total_fees_single_path() {
                let route = Route {
-                       paths: vec![vec![
+                       paths: vec![Path { hops: vec![
                                RouteHop {
                                        pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
@@ -5215,7 +5349,7 @@ mod tests {
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
                                        short_channel_id: 0, fee_msat: 225, cltv_expiry_delta: 0
                                },
-                       ]],
+                       ], blinded_tail: None }],
                        payment_params: None,
                };
 
@@ -5226,7 +5360,7 @@ mod tests {
        #[test]
        fn total_fees_multi_path() {
                let route = Route {
-                       paths: vec![vec![
+                       paths: vec![Path { hops: vec![
                                RouteHop {
                                        pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
@@ -5237,7 +5371,7 @@ mod tests {
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
                                        short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0
                                },
-                       ],vec![
+                       ], blinded_tail: None }, Path { hops: vec![
                                RouteHop {
                                        pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
@@ -5248,7 +5382,7 @@ mod tests {
                                        channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
                                        short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0
                                },
-                       ]],
+                       ], blinded_tail: None }],
                        payment_params: None,
                };
 
@@ -5282,7 +5416,7 @@ mod tests {
                let keys_manager = ln_test_utils::TestKeysInterface::new(&[0u8; 32], Network::Testnet);
                let random_seed_bytes = keys_manager.get_secure_random_bytes();
                let route = get_route(&our_id, &feasible_payment_params, &network_graph, None, 100, 0, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
                assert_ne!(path.len(), 0);
 
                // But not if we exclude all paths on the basis of their accumulated CLTV delta
@@ -5317,12 +5451,12 @@ mod tests {
                assert!(get_route(&our_id, &payment_params, &network_graph, None, 100, 0, Arc::clone(&logger), &scorer, &random_seed_bytes).is_ok());
                loop {
                        if let Ok(route) = get_route(&our_id, &payment_params, &network_graph, None, 100, 0, Arc::clone(&logger), &scorer, &random_seed_bytes) {
-                               for chan in route.paths[0].iter() {
+                               for chan in route.paths[0].hops.iter() {
                                        assert!(!payment_params.previously_failed_channels.contains(&chan.short_channel_id));
                                }
                                let victim = (u64::from_ne_bytes(random_seed_bytes[0..8].try_into().unwrap()) as usize)
-                                       % route.paths[0].len();
-                               payment_params.previously_failed_channels.push(route.paths[0][victim].short_channel_id);
+                                       % route.paths[0].hops.len();
+                               payment_params.previously_failed_channels.push(route.paths[0].hops[victim].short_channel_id);
                        } else { break; }
                }
        }
@@ -5341,7 +5475,7 @@ mod tests {
                let feasible_payment_params = PaymentParameters::from_node_id(nodes[18], 0);
                let route = get_route(&our_id, &feasible_payment_params, &network_graph, None, 100, 0,
                        Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
-               let path = route.paths[0].iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
+               let path = route.paths[0].hops.iter().map(|hop| hop.short_channel_id).collect::<Vec<_>>();
                assert!(path.len() == MAX_PATH_LENGTH_ESTIMATE.into());
 
                // But we can't create a path surpassing the MAX_PATH_LENGTH_ESTIMATE limit.
@@ -5369,12 +5503,12 @@ mod tests {
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                assert_eq!(route.paths.len(), 1);
 
-               let cltv_expiry_deltas_before = route.paths[0].iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
+               let cltv_expiry_deltas_before = route.paths[0].hops.iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
 
                // Check whether the offset added to the last hop by default is in [1 .. DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA]
                let mut route_default = route.clone();
                add_random_cltv_offset(&mut route_default, &payment_params, &network_graph.read_only(), &random_seed_bytes);
-               let cltv_expiry_deltas_default = route_default.paths[0].iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
+               let cltv_expiry_deltas_default = route_default.paths[0].hops.iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
                assert_eq!(cltv_expiry_deltas_before.split_last().unwrap().1, cltv_expiry_deltas_default.split_last().unwrap().1);
                assert!(cltv_expiry_deltas_default.last() > cltv_expiry_deltas_before.last());
                assert!(cltv_expiry_deltas_default.last().unwrap() <= &DEFAULT_MAX_TOTAL_CLTV_EXPIRY_DELTA);
@@ -5384,7 +5518,7 @@ mod tests {
                let limited_max_total_cltv_expiry_delta = cltv_expiry_deltas_before.iter().sum();
                let limited_payment_params = payment_params.with_max_total_cltv_expiry_delta(limited_max_total_cltv_expiry_delta);
                add_random_cltv_offset(&mut route_limited, &limited_payment_params, &network_graph.read_only(), &random_seed_bytes);
-               let cltv_expiry_deltas_limited = route_limited.paths[0].iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
+               let cltv_expiry_deltas_limited = route_limited.paths[0].hops.iter().map(|h| h.cltv_expiry_delta).collect::<Vec<u32>>();
                assert_eq!(cltv_expiry_deltas_before, cltv_expiry_deltas_limited);
        }
 
@@ -5412,11 +5546,11 @@ mod tests {
                        let mut random_bytes = [0u8; ::core::mem::size_of::<usize>()];
 
                        prng.process_in_place(&mut random_bytes);
-                       let random_path_index = usize::from_be_bytes(random_bytes).wrapping_rem(p.len());
-                       let observation_point = NodeId::from_pubkey(&p.get(random_path_index).unwrap().pubkey);
+                       let random_path_index = usize::from_be_bytes(random_bytes).wrapping_rem(p.hops.len());
+                       let observation_point = NodeId::from_pubkey(&p.hops.get(random_path_index).unwrap().pubkey);
 
                        // 2. Calculate what CLTV expiry delta we would observe there
-                       let observed_cltv_expiry_delta: u32 = p[random_path_index..].iter().map(|h| h.cltv_expiry_delta).sum();
+                       let observed_cltv_expiry_delta: u32 = p.hops[random_path_index..].iter().map(|h| h.cltv_expiry_delta).sum();
 
                        // 3. Starting from the observation point, find candidate paths
                        let mut candidates: VecDeque<(NodeId, Vec<u32>)> = VecDeque::new();
@@ -5467,8 +5601,8 @@ mod tests {
                let hops = [nodes[1], nodes[2], nodes[4], nodes[3]];
                let route = build_route_from_hops_internal(&our_id, &hops, &payment_params,
                         &network_graph, 100, 0, Arc::clone(&logger), &random_seed_bytes).unwrap();
-               let route_hop_pubkeys = route.paths[0].iter().map(|hop| hop.pubkey).collect::<Vec<_>>();
-               assert_eq!(hops.len(), route.paths[0].len());
+               let route_hop_pubkeys = route.paths[0].hops.iter().map(|hop| hop.pubkey).collect::<Vec<_>>();
+               assert_eq!(hops.len(), route.paths[0].hops.len());
                for (idx, hop_pubkey) in hops.iter().enumerate() {
                        assert!(*hop_pubkey == route_hop_pubkeys[idx]);
                }
@@ -5515,8 +5649,8 @@ mod tests {
                // 100,000 sats is less than the available liquidity on each channel, set above.
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100_000_000, 42, Arc::clone(&logger), &scorer, &random_seed_bytes).unwrap();
                assert_eq!(route.paths.len(), 2);
-               assert!((route.paths[0][1].short_channel_id == 4 && route.paths[1][1].short_channel_id == 13) ||
-                       (route.paths[1][1].short_channel_id == 4 && route.paths[0][1].short_channel_id == 13));
+               assert!((route.paths[0].hops[1].short_channel_id == 4 && route.paths[1].hops[1].short_channel_id == 13) ||
+                       (route.paths[1].hops[1].short_channel_id == 4 && route.paths[0].hops[1].short_channel_id == 13));
        }
 
        #[cfg(not(feature = "no-std"))]
@@ -5641,6 +5775,151 @@ mod tests {
                let route = get_route(&our_id, &payment_params, &network_graph.read_only(), None, 100, 42, Arc::clone(&logger), &scorer, &random_seed_bytes);
                assert!(route.is_ok());
        }
+
+       #[test]
+       fn blinded_route_ser() {
+               let blinded_path_1 = BlindedPath {
+                       introduction_node_id: ln_test_utils::pubkey(42),
+                       blinding_point: ln_test_utils::pubkey(43),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(44), encrypted_payload: Vec::new() },
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(45), encrypted_payload: Vec::new() }
+                       ],
+               };
+               let blinded_path_2 = BlindedPath {
+                       introduction_node_id: ln_test_utils::pubkey(46),
+                       blinding_point: ln_test_utils::pubkey(47),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(48), encrypted_payload: Vec::new() },
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(49), encrypted_payload: Vec::new() }
+                       ],
+               };
+               // (De)serialize a Route with 1 blinded path out of two total paths.
+               let mut route = Route { paths: vec![Path {
+                       hops: vec![RouteHop {
+                               pubkey: ln_test_utils::pubkey(50),
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 42,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 100,
+                               cltv_expiry_delta: 0,
+                       }],
+                       blinded_tail: Some(BlindedTail {
+                               hops: blinded_path_1.blinded_hops,
+                               blinding_point: blinded_path_1.blinding_point,
+                               excess_final_cltv_expiry_delta: 40,
+                               final_value_msat: 100,
+                       })}, Path {
+                       hops: vec![RouteHop {
+                               pubkey: ln_test_utils::pubkey(51),
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 43,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 100,
+                               cltv_expiry_delta: 0,
+                       }], blinded_tail: None }],
+                       payment_params: None,
+               };
+               let encoded_route = route.encode();
+               let decoded_route: Route = Readable::read(&mut Cursor::new(&encoded_route[..])).unwrap();
+               assert_eq!(decoded_route.paths[0].blinded_tail, route.paths[0].blinded_tail);
+               assert_eq!(decoded_route.paths[1].blinded_tail, route.paths[1].blinded_tail);
+
+               // (De)serialize a Route with two paths, each containing a blinded tail.
+               route.paths[1].blinded_tail = Some(BlindedTail {
+                       hops: blinded_path_2.blinded_hops,
+                       blinding_point: blinded_path_2.blinding_point,
+                       excess_final_cltv_expiry_delta: 41,
+                       final_value_msat: 101,
+               });
+               let encoded_route = route.encode();
+               let decoded_route: Route = Readable::read(&mut Cursor::new(&encoded_route[..])).unwrap();
+               assert_eq!(decoded_route.paths[0].blinded_tail, route.paths[0].blinded_tail);
+               assert_eq!(decoded_route.paths[1].blinded_tail, route.paths[1].blinded_tail);
+       }
+
+       #[test]
+       fn blinded_path_inflight_processing() {
+               // Ensure we'll score the channel that's inbound to a blinded path's introduction node, and
+               // account for the blinded tail's final amount_msat.
+               let mut inflight_htlcs = InFlightHtlcs::new();
+               let blinded_path = BlindedPath {
+                       introduction_node_id: ln_test_utils::pubkey(43),
+                       blinding_point: ln_test_utils::pubkey(48),
+                       blinded_hops: vec![BlindedHop { blinded_node_id: ln_test_utils::pubkey(49), encrypted_payload: Vec::new() }],
+               };
+               let path = Path {
+                       hops: vec![RouteHop {
+                               pubkey: ln_test_utils::pubkey(42),
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 42,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 100,
+                               cltv_expiry_delta: 0,
+                       },
+                       RouteHop {
+                               pubkey: blinded_path.introduction_node_id,
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 43,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 1,
+                               cltv_expiry_delta: 0,
+                       }],
+                       blinded_tail: Some(BlindedTail {
+                               hops: blinded_path.blinded_hops,
+                               blinding_point: blinded_path.blinding_point,
+                               excess_final_cltv_expiry_delta: 0,
+                               final_value_msat: 200,
+                       }),
+               };
+               inflight_htlcs.process_path(&path, ln_test_utils::pubkey(44));
+               assert_eq!(*inflight_htlcs.0.get(&(42, true)).unwrap(), 301);
+               assert_eq!(*inflight_htlcs.0.get(&(43, false)).unwrap(), 201);
+       }
+
+       #[test]
+       fn blinded_path_cltv_shadow_offset() {
+               // Make sure we add a shadow offset when sending to blinded paths.
+               let blinded_path = BlindedPath {
+                       introduction_node_id: ln_test_utils::pubkey(43),
+                       blinding_point: ln_test_utils::pubkey(44),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(45), encrypted_payload: Vec::new() },
+                               BlindedHop { blinded_node_id: ln_test_utils::pubkey(46), encrypted_payload: Vec::new() }
+                       ],
+               };
+               let mut route = Route { paths: vec![Path {
+                       hops: vec![RouteHop {
+                               pubkey: ln_test_utils::pubkey(42),
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 42,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 100,
+                               cltv_expiry_delta: 0,
+                       },
+                       RouteHop {
+                               pubkey: blinded_path.introduction_node_id,
+                               node_features: NodeFeatures::empty(),
+                               short_channel_id: 43,
+                               channel_features: ChannelFeatures::empty(),
+                               fee_msat: 1,
+                               cltv_expiry_delta: 0,
+                       }
+                       ],
+                       blinded_tail: Some(BlindedTail {
+                               hops: blinded_path.blinded_hops,
+                               blinding_point: blinded_path.blinding_point,
+                               excess_final_cltv_expiry_delta: 0,
+                               final_value_msat: 200,
+                       }),
+               }], payment_params: None};
+
+               let payment_params = PaymentParameters::from_node_id(ln_test_utils::pubkey(47), 18);
+               let (_, network_graph, _, _, _) = build_line_graph();
+               add_random_cltv_offset(&mut route, &payment_params, &network_graph.read_only(), &[0; 32]);
+               assert_eq!(route.paths[0].blinded_tail.as_ref().unwrap().excess_final_cltv_expiry_delta, 40);
+               assert_eq!(route.paths[0].hops.last().unwrap().cltv_expiry_delta, 40);
+       }
 }
 
 #[cfg(all(test, not(feature = "no-std")))]
@@ -5811,12 +6090,12 @@ mod benches {
                        let amount = route.get_total_amount();
                        if amount < 250_000 {
                                for path in route.paths {
-                                       scorer.payment_path_successful(&path.iter().collect::<Vec<_>>());
+                                       scorer.payment_path_successful(&path);
                                }
                        } else if amount > 750_000 {
                                for path in route.paths {
-                                       let short_channel_id = path[path.len() / 2].short_channel_id;
-                                       scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), short_channel_id);
+                                       let short_channel_id = path.hops[path.hops.len() / 2].short_channel_id;
+                                       scorer.payment_path_failed(&path, short_channel_id);
                                }
                        }
                }
index 4d342562bea75afbe091af9244584dcbb2124ffb..e60e4879b3d5e25ca121ed6d1a543b120317282f 100644 (file)
@@ -56,7 +56,7 @@
 
 use crate::ln::msgs::DecodeError;
 use crate::routing::gossip::{EffectiveCapacity, NetworkGraph, NodeId};
-use crate::routing::router::RouteHop;
+use crate::routing::router::Path;
 use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer};
 use crate::util::logger::Logger;
 use crate::util::time::Time;
@@ -99,16 +99,16 @@ pub trait Score $(: $supertrait)* {
        ) -> u64;
 
        /// Handles updating channel penalties after failing to route through a channel.
-       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64);
+       fn payment_path_failed(&mut self, path: &Path, short_channel_id: u64);
 
        /// Handles updating channel penalties after successfully routing along a path.
-       fn payment_path_successful(&mut self, path: &[&RouteHop]);
+       fn payment_path_successful(&mut self, path: &Path);
 
        /// Handles updating channel penalties after a probe over the given path failed.
-       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64);
+       fn probe_failed(&mut self, path: &Path, short_channel_id: u64);
 
        /// Handles updating channel penalties after a probe over the given path succeeded.
-       fn probe_successful(&mut self, path: &[&RouteHop]);
+       fn probe_successful(&mut self, path: &Path);
 }
 
 impl<S: Score, T: DerefMut<Target=S> $(+ $supertrait)*> Score for T {
@@ -118,19 +118,19 @@ impl<S: Score, T: DerefMut<Target=S> $(+ $supertrait)*> Score for T {
                self.deref().channel_penalty_msat(short_channel_id, source, target, usage)
        }
 
-       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn payment_path_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.deref_mut().payment_path_failed(path, short_channel_id)
        }
 
-       fn payment_path_successful(&mut self, path: &[&RouteHop]) {
+       fn payment_path_successful(&mut self, path: &Path) {
                self.deref_mut().payment_path_successful(path)
        }
 
-       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn probe_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.deref_mut().probe_failed(path, short_channel_id)
        }
 
-       fn probe_successful(&mut self, path: &[&RouteHop]) {
+       fn probe_successful(&mut self, path: &Path) {
                self.deref_mut().probe_successful(path)
        }
 }
@@ -195,16 +195,16 @@ impl<'a, T: Score + 'a> Score for MultiThreadedScoreLock<'a, T> {
        fn channel_penalty_msat(&self, scid: u64, source: &NodeId, target: &NodeId, usage: ChannelUsage) -> u64 {
                self.0.channel_penalty_msat(scid, source, target, usage)
        }
-       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn payment_path_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.0.payment_path_failed(path, short_channel_id)
        }
-       fn payment_path_successful(&mut self, path: &[&RouteHop]) {
+       fn payment_path_successful(&mut self, path: &Path) {
                self.0.payment_path_successful(path)
        }
-       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn probe_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.0.probe_failed(path, short_channel_id)
        }
-       fn probe_successful(&mut self, path: &[&RouteHop]) {
+       fn probe_successful(&mut self, path: &Path) {
                self.0.probe_successful(path)
        }
 }
@@ -290,13 +290,13 @@ impl Score for FixedPenaltyScorer {
                self.penalty_msat
        }
 
-       fn payment_path_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
+       fn payment_path_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
 
-       fn payment_path_successful(&mut self, _path: &[&RouteHop]) {}
+       fn payment_path_successful(&mut self, _path: &Path) {}
 
-       fn probe_failed(&mut self, _path: &[&RouteHop], _short_channel_id: u64) {}
+       fn probe_failed(&mut self, _path: &Path, _short_channel_id: u64) {}
 
-       fn probe_successful(&mut self, _path: &[&RouteHop]) {}
+       fn probe_successful(&mut self, _path: &Path) {}
 }
 
 impl Writeable for FixedPenaltyScorer {
@@ -1233,11 +1233,11 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, T: Time> Score for Probabilis
                        .saturating_add(base_penalty_msat)
        }
 
-       fn payment_path_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
-               let amount_msat = path.split_last().map(|(hop, _)| hop.fee_msat).unwrap_or(0);
+       fn payment_path_failed(&mut self, path: &Path, short_channel_id: u64) {
+               let amount_msat = path.final_value_msat();
                log_trace!(self.logger, "Scoring path through to SCID {} as having failed at {} msat", short_channel_id, amount_msat);
                let network_graph = self.network_graph.read_only();
-               for (hop_idx, hop) in path.iter().enumerate() {
+               for (hop_idx, hop) in path.hops.iter().enumerate() {
                        let target = NodeId::from_pubkey(&hop.pubkey);
                        let channel_directed_from_source = network_graph.channels()
                                .get(&hop.short_channel_id)
@@ -1272,12 +1272,12 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, T: Time> Score for Probabilis
                }
        }
 
-       fn payment_path_successful(&mut self, path: &[&RouteHop]) {
-               let amount_msat = path.split_last().map(|(hop, _)| hop.fee_msat).unwrap_or(0);
+       fn payment_path_successful(&mut self, path: &Path) {
+               let amount_msat = path.final_value_msat();
                log_trace!(self.logger, "Scoring path through SCID {} as having succeeded at {} msat.",
-                       path.split_last().map(|(hop, _)| hop.short_channel_id).unwrap_or(0), amount_msat);
+                       path.hops.split_last().map(|(hop, _)| hop.short_channel_id).unwrap_or(0), amount_msat);
                let network_graph = self.network_graph.read_only();
-               for hop in path {
+               for hop in &path.hops {
                        let target = NodeId::from_pubkey(&hop.pubkey);
                        let channel_directed_from_source = network_graph.channels()
                                .get(&hop.short_channel_id)
@@ -1298,11 +1298,11 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, T: Time> Score for Probabilis
                }
        }
 
-       fn probe_failed(&mut self, path: &[&RouteHop], short_channel_id: u64) {
+       fn probe_failed(&mut self, path: &Path, short_channel_id: u64) {
                self.payment_path_failed(path, short_channel_id)
        }
 
-       fn probe_successful(&mut self, path: &[&RouteHop]) {
+       fn probe_successful(&mut self, path: &Path) {
                self.payment_path_failed(path, u64::max_value())
        }
 }
@@ -1702,6 +1702,7 @@ impl<T: Time> Readable for ChannelLiquidity<T> {
 #[cfg(test)]
 mod tests {
        use super::{ChannelLiquidity, HistoricalBucketRangeTracker, ProbabilisticScoringParameters, ProbabilisticScorerUsingTime};
+       use crate::blinded_path::{BlindedHop, BlindedPath};
        use crate::util::config::UserConfig;
        use crate::util::time::Time;
        use crate::util::time::tests::SinceEpoch;
@@ -1709,10 +1710,10 @@ mod tests {
        use crate::ln::channelmanager;
        use crate::ln::msgs::{ChannelAnnouncement, ChannelUpdate, UnsignedChannelAnnouncement, UnsignedChannelUpdate};
        use crate::routing::gossip::{EffectiveCapacity, NetworkGraph, NodeId};
-       use crate::routing::router::RouteHop;
+       use crate::routing::router::{BlindedTail, Path, RouteHop};
        use crate::routing::scoring::{ChannelUsage, Score};
        use crate::util::ser::{ReadableArgs, Writeable};
-       use crate::util::test_utils::TestLogger;
+       use crate::util::test_utils::{self, TestLogger};
 
        use bitcoin::blockdata::constants::genesis_block;
        use bitcoin::hashes::Hash;
@@ -1858,12 +1859,14 @@ mod tests {
                }
        }
 
-       fn payment_path_for_amount(amount_msat: u64) -> Vec<RouteHop> {
-               vec![
-                       path_hop(source_pubkey(), 41, 1),
-                       path_hop(target_pubkey(), 42, 2),
-                       path_hop(recipient_pubkey(), 43, amount_msat),
-               ]
+       fn payment_path_for_amount(amount_msat: u64) -> Path {
+               Path {
+                       hops: vec![
+                               path_hop(source_pubkey(), 41, 1),
+                               path_hop(target_pubkey(), 42, 2),
+                               path_hop(recipient_pubkey(), 43, amount_msat),
+                       ], blinded_tail: None,
+               }
        }
 
        #[test]
@@ -2161,10 +2164,10 @@ mod tests {
 
                assert_eq!(scorer.channel_penalty_msat(41, &sender, &source, usage), 301);
 
-               scorer.payment_path_failed(&failed_path.iter().collect::<Vec<_>>(), 41);
+               scorer.payment_path_failed(&failed_path, 41);
                assert_eq!(scorer.channel_penalty_msat(41, &sender, &source, usage), 301);
 
-               scorer.payment_path_successful(&successful_path.iter().collect::<Vec<_>>());
+               scorer.payment_path_successful(&successful_path);
                assert_eq!(scorer.channel_penalty_msat(41, &sender, &source, usage), 301);
        }
 
@@ -2192,7 +2195,7 @@ mod tests {
                let usage = ChannelUsage { amount_msat: 750, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 602);
 
-               scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&path, 43);
 
                let usage = ChannelUsage { amount_msat: 250, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
@@ -2227,7 +2230,7 @@ mod tests {
                let usage = ChannelUsage { amount_msat: 750, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 602);
 
-               scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&path, 42);
 
                let usage = ChannelUsage { amount_msat: 250, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
@@ -2287,7 +2290,7 @@ mod tests {
                assert_eq!(scorer.channel_penalty_msat(43, &node_b, &node_c, usage), 128);
                assert_eq!(scorer.channel_penalty_msat(44, &node_c, &node_d, usage), 128);
 
-               scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&Path { hops: path, blinded_tail: None }, 43);
 
                assert_eq!(scorer.channel_penalty_msat(42, &node_a, &node_b, usage), 80);
                // Note that a default liquidity bound is used for B -> C as no channel exists
@@ -2313,13 +2316,12 @@ mod tests {
                        inflight_htlc_msat: 0,
                        effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
-               let path = payment_path_for_amount(500);
 
                assert_eq!(scorer.channel_penalty_msat(41, &sender, &source, usage), 128);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 128);
                assert_eq!(scorer.channel_penalty_msat(43, &target, &recipient, usage), 128);
 
-               scorer.payment_path_successful(&path.iter().collect::<Vec<_>>());
+               scorer.payment_path_successful(&payment_path_for_amount(500));
 
                assert_eq!(scorer.channel_penalty_msat(41, &sender, &source, usage), 128);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
@@ -2349,8 +2351,8 @@ mod tests {
                let usage = ChannelUsage { amount_msat: 1_023, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2_000);
 
-               scorer.payment_path_failed(&payment_path_for_amount(768).iter().collect::<Vec<_>>(), 42);
-               scorer.payment_path_failed(&payment_path_for_amount(128).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(768), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(128), 43);
 
                let usage = ChannelUsage { amount_msat: 128, ..usage };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
@@ -2425,7 +2427,7 @@ mod tests {
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 125);
 
-               scorer.payment_path_failed(&payment_path_for_amount(512).iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(512), 42);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 281);
 
                // An unchecked right shift 64 bits or more in DirectedChannelLiquidity::decayed_offset_msat
@@ -2458,8 +2460,8 @@ mod tests {
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
 
                // More knowledge gives higher confidence (256, 768), meaning a lower penalty.
-               scorer.payment_path_failed(&payment_path_for_amount(768).iter().collect::<Vec<_>>(), 42);
-               scorer.payment_path_failed(&payment_path_for_amount(256).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(768), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(256), 43);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 281);
 
                // Decaying knowledge gives less confidence (128, 896), meaning a higher penalty.
@@ -2468,12 +2470,12 @@ mod tests {
 
                // Reducing the upper bound gives more confidence (128, 832) that the payment amount (512)
                // is closer to the upper bound, meaning a higher penalty.
-               scorer.payment_path_successful(&payment_path_for_amount(64).iter().collect::<Vec<_>>());
+               scorer.payment_path_successful(&payment_path_for_amount(64));
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 331);
 
                // Increasing the lower bound gives more confidence (256, 832) that the payment amount (512)
                // is closer to the lower bound, meaning a lower penalty.
-               scorer.payment_path_failed(&payment_path_for_amount(256).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(256), 43);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 245);
 
                // Further decaying affects the lower bound more than the upper bound (128, 928).
@@ -2500,13 +2502,13 @@ mod tests {
                        effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
 
-               scorer.payment_path_failed(&payment_path_for_amount(500).iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(500), 42);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), u64::max_value());
 
                SinceEpoch::advance(Duration::from_secs(10));
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 473);
 
-               scorer.payment_path_failed(&payment_path_for_amount(250).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(250), 43);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
 
                let mut serialized_scorer = Vec::new();
@@ -2537,7 +2539,7 @@ mod tests {
                        effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
 
-               scorer.payment_path_failed(&payment_path_for_amount(500).iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(500), 42);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), u64::max_value());
 
                let mut serialized_scorer = Vec::new();
@@ -2550,7 +2552,7 @@ mod tests {
                        <ProbabilisticScorer>::read(&mut serialized_scorer, (params, &network_graph, &logger)).unwrap();
                assert_eq!(deserialized_scorer.channel_penalty_msat(42, &source, &target, usage), 473);
 
-               scorer.payment_path_failed(&payment_path_for_amount(250).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(250), 43);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
 
                SinceEpoch::advance(Duration::from_secs(10));
@@ -2773,7 +2775,7 @@ mod tests {
                assert_eq!(scorer.historical_estimated_channel_liquidity_probabilities(42, &target),
                        None);
 
-               scorer.payment_path_failed(&payment_path_for_amount(1).iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(1), 42);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2048);
                // The "it failed" increment is 32, where the probability should lie fully in the first
                // octile.
@@ -2782,7 +2784,7 @@ mod tests {
 
                // Even after we tell the scorer we definitely have enough available liquidity, it will
                // still remember that there was some failure in the past, and assign a non-0 penalty.
-               scorer.payment_path_failed(&payment_path_for_amount(1000).iter().collect::<Vec<_>>(), 43);
+               scorer.payment_path_failed(&payment_path_for_amount(1000), 43);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 198);
                // The first octile should be decayed just slightly and the last octile has a new point.
                assert_eq!(scorer.historical_estimated_channel_liquidity_probabilities(42, &target),
@@ -2802,7 +2804,7 @@ mod tests {
                        inflight_htlc_msat: 1024,
                        effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_024 },
                };
-               scorer.payment_path_failed(&payment_path_for_amount(1).iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&payment_path_for_amount(1), 42);
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 409);
 
                let usage = ChannelUsage {
@@ -2822,7 +2824,7 @@ mod tests {
                        path_hop(source_pubkey(), 42, 1),
                        path_hop(sender_pubkey(), 41, 0),
                ];
-               scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), 42);
+               scorer.payment_path_failed(&Path { hops: path, blinded_tail: None }, 42);
        }
 
        #[test]
@@ -2869,4 +2871,54 @@ mod tests {
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
        }
+
+       #[test]
+       fn scores_with_blinded_path() {
+               // Make sure we'll account for a blinded path's final_value_msat in scoring
+               let logger = TestLogger::new();
+               let network_graph = network_graph(&logger);
+               let params = ProbabilisticScoringParameters {
+                       liquidity_penalty_multiplier_msat: 1_000,
+                       liquidity_offset_half_life: Duration::from_secs(10),
+                       ..ProbabilisticScoringParameters::zero_penalty()
+               };
+               let mut scorer = ProbabilisticScorer::new(params, &network_graph, &logger);
+               let source = source_node_id();
+               let target = target_node_id();
+               let usage = ChannelUsage {
+                       amount_msat: 512,
+                       inflight_htlc_msat: 0,
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_000 },
+               };
+               assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
+
+               let mut path = payment_path_for_amount(768);
+               let recipient_hop = path.hops.pop().unwrap();
+               let blinded_path = BlindedPath {
+                       introduction_node_id: path.hops.last().as_ref().unwrap().pubkey,
+                       blinding_point: test_utils::pubkey(42),
+                       blinded_hops: vec![
+                               BlindedHop { blinded_node_id: test_utils::pubkey(44), encrypted_payload: Vec::new() }
+                       ],
+               };
+               path.blinded_tail = Some(BlindedTail {
+                       hops: blinded_path.blinded_hops,
+                       blinding_point: blinded_path.blinding_point,
+                       excess_final_cltv_expiry_delta: recipient_hop.cltv_expiry_delta,
+                       final_value_msat: recipient_hop.fee_msat,
+               });
+
+               // Check the liquidity before and after scoring payment failures to ensure the blinded path's
+               // final value is taken into account.
+               assert!(scorer.channel_liquidities.get(&42).is_none());
+
+               scorer.payment_path_failed(&path, 42);
+               path.blinded_tail.as_mut().unwrap().final_value_msat = 256;
+               scorer.payment_path_failed(&path, 43);
+
+               let liquidity = scorer.channel_liquidities.get(&42).unwrap()
+                       .as_directed(&source, &target, 0, 1_000, &scorer.params);
+               assert_eq!(liquidity.min_liquidity_msat(), 256);
+               assert_eq!(liquidity.max_liquidity_msat(), 768);
+       }
 }
index 68264207f589e249fd90bde0a6d00298c2621b74..d0e44eb214cf4fe3d43a5a8c52b6b6e340a9f2b9 100644 (file)
@@ -7,7 +7,7 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-use crate::routing::gossip::{NetworkGraph, P2PGossipSync};
+use crate::routing::gossip::{NetworkGraph, NodeAlias, P2PGossipSync};
 use crate::ln::features::{ChannelFeatures, NodeFeatures};
 use crate::ln::msgs::{UnsignedChannelAnnouncement, ChannelAnnouncement, RoutingMessageHandler,
        NodeAnnouncement, UnsignedNodeAnnouncement, ChannelUpdate, UnsignedChannelUpdate, MAX_VALUE_MSAT};
@@ -72,7 +72,7 @@ pub(super) fn add_or_update_node(
                timestamp,
                node_id,
                rgb: [0; 3],
-               alias: [0; 32],
+               alias: NodeAlias([0; 32]),
                addresses: Vec::new(),
                excess_address_data: Vec::new(),
                excess_data: Vec::new(),
index 43338a15969eecce33502e6238acc0691e73e105..8733def13d95d26dc3d9c446cb5851d39c3b2ca3 100644 (file)
@@ -125,7 +125,7 @@ pub struct UtxoFuture {
 
 /// A trivial implementation of [`UtxoLookup`] which is used to call back into the network graph
 /// once we have a concrete resolution of a request.
-struct UtxoResolver(Result<TxOut, UtxoLookupError>);
+pub(crate) struct UtxoResolver(Result<TxOut, UtxoLookupError>);
 impl UtxoLookup for UtxoResolver {
        fn get_utxo(&self, _genesis_hash: &BlockHash, _short_channel_id: u64) -> UtxoResult {
                UtxoResult::Sync(self.0.clone())
index 11557be82afec418be0abce3b38117ce04582fd0..b9f015af65697bedea34edaabcede1c9e559bdfd 100644 (file)
@@ -12,6 +12,8 @@ use std::sync::RwLockReadGuard as StdRwLockReadGuard;
 use std::sync::RwLockWriteGuard as StdRwLockWriteGuard;
 use std::sync::Condvar as StdCondvar;
 
+pub use std::sync::WaitTimeoutResult;
+
 use crate::prelude::HashMap;
 
 use super::{LockTestExt, LockHeldState};
@@ -35,15 +37,19 @@ impl Condvar {
                Condvar { inner: StdCondvar::new() }
        }
 
-       pub fn wait<'a, T>(&'a self, guard: MutexGuard<'a, T>) -> LockResult<MutexGuard<'a, T>> {
+       pub fn wait_while<'a, T, F: FnMut(&mut T) -> bool>(&'a self, guard: MutexGuard<'a, T>, condition: F)
+       -> LockResult<MutexGuard<'a, T>> {
                let mutex: &'a Mutex<T> = guard.mutex;
-               self.inner.wait(guard.into_inner()).map(|lock| MutexGuard { mutex, lock }).map_err(|_| ())
+               self.inner.wait_while(guard.into_inner(), condition).map(|lock| MutexGuard { mutex, lock })
+                       .map_err(|_| ())
        }
 
        #[allow(unused)]
-       pub fn wait_timeout<'a, T>(&'a self, guard: MutexGuard<'a, T>, dur: Duration) -> LockResult<(MutexGuard<'a, T>, ())> {
+       pub fn wait_timeout_while<'a, T, F: FnMut(&mut T) -> bool>(&'a self, guard: MutexGuard<'a, T>, dur: Duration, condition: F)
+       -> LockResult<(MutexGuard<'a, T>, WaitTimeoutResult)> {
                let mutex = guard.mutex;
-               self.inner.wait_timeout(guard.into_inner(), dur).map(|(lock, _)| (MutexGuard { mutex, lock }, ())).map_err(|_| ())
+               self.inner.wait_timeout_while(guard.into_inner(), dur, condition).map_err(|_| ())
+                       .map(|(lock, e)| (MutexGuard { mutex, lock }, e))
        }
 
        pub fn notify_all(&self) { self.inner.notify_all(); }
index 17307997d8176cc2b82a0d9559307971f3e2c252..08d54a939be66ecccb83d30c0569d25f31a523e1 100644 (file)
@@ -1,30 +1,10 @@
 pub use ::alloc::sync::Arc;
 use core::ops::{Deref, DerefMut};
-use core::time::Duration;
 use core::cell::{RefCell, Ref, RefMut};
 use super::{LockTestExt, LockHeldState};
 
 pub type LockResult<Guard> = Result<Guard, ()>;
 
-pub struct Condvar {}
-
-impl Condvar {
-       pub fn new() -> Condvar {
-               Condvar { }
-       }
-
-       pub fn wait<'a, T>(&'a self, guard: MutexGuard<'a, T>) -> LockResult<MutexGuard<'a, T>> {
-               Ok(guard)
-       }
-
-       #[allow(unused)]
-       pub fn wait_timeout<'a, T>(&'a self, guard: MutexGuard<'a, T>, _dur: Duration) -> LockResult<(MutexGuard<'a, T>, ())> {
-               Ok((guard, ()))
-       }
-
-       pub fn notify_all(&self) {}
-}
-
 pub struct Mutex<T: ?Sized> {
        inner: RefCell<T>
 }
index ec34f5322890f2095fcc56f22d237dedc2b76a56..ba00158c635f5ef7fb85663b9bc8702b03695e28 100644 (file)
@@ -170,6 +170,19 @@ pub struct ChannelHandshakeConfig {
        /// [`DecodeError::InvalidValue`]: crate::ln::msgs::DecodeError::InvalidValue
        /// [`SIGHASH_SINGLE + update_fee Considered Harmful`]: https://lists.linuxfoundation.org/pipermail/lightning-dev/2020-September/002796.html
        pub negotiate_anchors_zero_fee_htlc_tx: bool,
+
+       /// The maximum number of HTLCs in-flight from our counterparty towards us at the same time.
+       ///
+       /// Increasing the value can help improve liquidity and stability in
+       /// routing at the cost of higher long term disk / DB usage.
+       ///
+       /// Note: Versions of LDK earlier than v0.0.115 will fail to read channels with a configuration
+       /// other than the default value.
+       ///
+       /// Default value: 50
+       /// Maximum value: 483, any values larger will be treated as 483.
+       ///                     This is the BOLT #2 spec limit on `max_accepted_htlcs`.
+       pub our_max_accepted_htlcs: u16,
 }
 
 impl Default for ChannelHandshakeConfig {
@@ -185,6 +198,7 @@ impl Default for ChannelHandshakeConfig {
                        their_channel_reserve_proportional_millionths: 10_000,
                        #[cfg(anchors)]
                        negotiate_anchors_zero_fee_htlc_tx: false,
+                       our_max_accepted_htlcs: 50,
                }
        }
 }
index 2f2d33b29f7ea53c0a2dda084ac7a139583c6345..7352542605070c7812201d0a10f35f65eaa3b3e4 100644 (file)
@@ -3,6 +3,10 @@ use bitcoin::hashes::hmac::{Hmac, HmacEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::{Message, Secp256k1, SecretKey, ecdsa::Signature, Signing};
 
+use crate::chain::keysinterface::EntropySource;
+
+use core::ops::Deref;
+
 macro_rules! hkdf_extract_expand {
        ($salt: expr, $ikm: expr) => {{
                let mut hmac = HmacEngine::<Sha256>::new($salt);
@@ -20,13 +24,18 @@ macro_rules! hkdf_extract_expand {
                let (k1, k2, _) = hkdf_extract_expand!($salt, $ikm);
                (k1, k2)
        }};
-       ($salt: expr, $ikm: expr, 3) => {{
+       ($salt: expr, $ikm: expr, 4) => {{
                let (k1, k2, prk) = hkdf_extract_expand!($salt, $ikm);
 
                let mut hmac = HmacEngine::<Sha256>::new(&prk[..]);
                hmac.input(&k2);
                hmac.input(&[3; 1]);
-               (k1, k2, Hmac::from_engine(hmac).into_inner())
+               let k3 = Hmac::from_engine(hmac).into_inner();
+
+               let mut hmac = HmacEngine::<Sha256>::new(&prk[..]);
+               hmac.input(&k3);
+               hmac.input(&[4; 1]);
+               (k1, k2, k3, Hmac::from_engine(hmac).into_inner())
        }}
 }
 
@@ -34,8 +43,8 @@ pub fn hkdf_extract_expand_twice(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32]
        hkdf_extract_expand!(salt, ikm, 2)
 }
 
-pub fn hkdf_extract_expand_thrice(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32], [u8; 32]) {
-       hkdf_extract_expand!(salt, ikm, 3)
+pub fn hkdf_extract_expand_4x(salt: &[u8], ikm: &[u8]) -> ([u8; 32], [u8; 32], [u8; 32], [u8; 32]) {
+       hkdf_extract_expand!(salt, ikm, 4)
 }
 
 #[inline]
@@ -46,3 +55,21 @@ pub fn sign<C: Signing>(ctx: &Secp256k1<C>, msg: &Message, sk: &SecretKey) -> Si
        let sig = ctx.sign_ecdsa(msg, sk);
        sig
 }
+
+#[inline]
+pub fn sign_with_aux_rand<C: Signing, ES: Deref>(
+       ctx: &Secp256k1<C>, msg: &Message, sk: &SecretKey, entropy_source: &ES
+) -> Signature where ES::Target: EntropySource {
+       #[cfg(feature = "grind_signatures")]
+       let sig = loop {
+               let sig = ctx.sign_ecdsa_with_noncedata(msg, sk, &entropy_source.get_secure_random_bytes());
+               if sig.serialize_compact()[0] < 0x80 {
+                       break sig;
+               }
+       };
+       #[cfg(all(not(feature = "grind_signatures"), not(feature = "_test_vectors")))]
+       let sig = ctx.sign_ecdsa_with_noncedata(msg, sk, &entropy_source.get_secure_random_bytes());
+       #[cfg(all(not(feature = "grind_signatures"), feature = "_test_vectors"))]
+       let sig = sign(ctx, msg, sk);
+       sig
+}
index 6e98272f3171d2bc47082c4171b6e4013800e911..a9018f3da90c89f412a4bb55ca438c55f8abec77 100644 (file)
@@ -65,9 +65,10 @@ impl<'a> core::fmt::Display for DebugRoute<'a> {
        fn fmt(&self, f: &mut core::fmt::Formatter) -> Result<(), core::fmt::Error> {
                for (idx, p) in self.0.paths.iter().enumerate() {
                        writeln!(f, "path {}:", idx)?;
-                       for h in p.iter() {
+                       for h in p.hops.iter() {
                                writeln!(f, " node_id: {}, short_channel_id: {}, fee_msat: {}, cltv_expiry_delta: {}", log_pubkey!(h.pubkey), h.short_channel_id, h.fee_msat, h.cltv_expiry_delta)?;
                        }
+                       writeln!(f, " blinded_tail: {:?}", p.blinded_tail)?;
                }
                Ok(())
        }
index 349105266e0aecaf7d4958c63dbe3f4f87103ae1..77ee33c4fa099de78ef6bac9f7c1cfd210586f73 100644 (file)
@@ -38,6 +38,8 @@ use bitcoin::hash_types::{Txid, BlockHash};
 use core::marker::Sized;
 use core::time::Duration;
 use crate::ln::msgs::DecodeError;
+#[cfg(taproot)]
+use crate::ln::msgs::PartialSignatureWithNonce;
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 
 use crate::util::byte_utils::{be48_to_array, slice_to_be48};
@@ -574,6 +576,7 @@ impl_array!(16); // for IPv6
 impl_array!(32); // for channel id & hmac
 impl_array!(PUBLIC_KEY_SIZE); // for PublicKey
 impl_array!(64); // for ecdsa::Signature and schnorr::Signature
+impl_array!(66); // for MuSig2 nonces
 impl_array!(1300); // for OnionPacket.hop_data
 
 impl Writeable for [u16; 8] {
@@ -593,7 +596,7 @@ impl Readable for [u16; 8] {
                r.read_exact(&mut buf)?;
                let mut res = [0u16; 8];
                for (idx, v) in res.iter_mut().enumerate() {
-                       *v = (buf[idx] as u16) << 8 | (buf[idx + 1] as u16)
+                       *v = (buf[idx*2] as u16) << 8 | (buf[idx*2 + 1] as u16)
                }
                Ok(res)
        }
@@ -746,7 +749,7 @@ where T: Readable + Eq + Hash
 }
 
 // Vectors
-macro_rules! impl_for_vec {
+macro_rules! impl_writeable_for_vec {
        ($ty: ty $(, $name: ident)*) => {
                impl<$($name : Writeable),*> Writeable for Vec<$ty> {
                        #[inline]
@@ -758,7 +761,10 @@ macro_rules! impl_for_vec {
                                Ok(())
                        }
                }
-
+       }
+}
+macro_rules! impl_readable_for_vec {
+       ($ty: ty $(, $name: ident)*) => {
                impl<$($name : Readable),*> Readable for Vec<$ty> {
                        #[inline]
                        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
@@ -774,6 +780,12 @@ macro_rules! impl_for_vec {
                }
        }
 }
+macro_rules! impl_for_vec {
+       ($ty: ty $(, $name: ident)*) => {
+               impl_writeable_for_vec!($ty $(, $name)*);
+               impl_readable_for_vec!($ty $(, $name)*);
+       }
+}
 
 impl Writeable for Vec<u8> {
        #[inline]
@@ -802,6 +814,8 @@ impl Readable for Vec<u8> {
 impl_for_vec!(ecdsa::Signature);
 impl_for_vec!(crate::ln::channelmanager::MonitorUpdateCompletionAction);
 impl_for_vec!((A, B), A, B);
+impl_writeable_for_vec!(&crate::routing::router::BlindedTail);
+impl_readable_for_vec!(crate::routing::router::BlindedTail);
 
 impl Writeable for Script {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
@@ -861,6 +875,39 @@ impl Readable for SecretKey {
        }
 }
 
+#[cfg(taproot)]
+impl Writeable for musig2::types::PublicNonce {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.serialize().write(w)
+       }
+}
+
+#[cfg(taproot)]
+impl Readable for musig2::types::PublicNonce {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let buf: [u8; PUBLIC_KEY_SIZE * 2] = Readable::read(r)?;
+               musig2::types::PublicNonce::from_slice(&buf).map_err(|_| DecodeError::InvalidValue)
+       }
+}
+
+#[cfg(taproot)]
+impl Writeable for PartialSignatureWithNonce {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.0.serialize().write(w)?;
+               self.1.write(w)
+       }
+}
+
+#[cfg(taproot)]
+impl Readable for PartialSignatureWithNonce {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let partial_signature_buf: [u8; SECRET_KEY_SIZE] = Readable::read(r)?;
+               let partial_signature = musig2::types::PartialSignature::from_slice(&partial_signature_buf).map_err(|_| DecodeError::InvalidValue)?;
+               let public_nonce: musig2::types::PublicNonce = Readable::read(r)?;
+               Ok(PartialSignatureWithNonce(partial_signature, public_nonce))
+       }
+}
+
 impl Writeable for Sha256dHash {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                w.write_all(&self[..])
@@ -1251,6 +1298,7 @@ impl Readable for Duration {
 #[cfg(test)]
 mod tests {
        use core::convert::TryFrom;
+       use bitcoin::secp256k1::ecdsa;
        use crate::util::ser::{Readable, Hostname, Writeable};
 
        #[test]
@@ -1273,11 +1321,22 @@ mod tests {
                assert_eq!(Hostname::read(&mut buf.as_slice()).unwrap().as_str(), "test");
        }
 
+       #[test]
+       /// Taproot will likely fill legacy signature fields with all 0s.
+       /// This test ensures that doing so won't break serialization.
+       fn null_signature_codec() {
+               let buffer = vec![0u8; 64];
+               let mut cursor = crate::io::Cursor::new(buffer.clone());
+               let signature = ecdsa::Signature::read(&mut cursor).unwrap();
+               let serialization = signature.serialize_compact();
+               assert_eq!(buffer, serialization.to_vec())
+       }
+
        #[test]
        fn bigsize_encoding_decoding() {
                let values = vec![0, 252, 253, 65535, 65536, 4294967295, 4294967296, 18446744073709551615];
                let bytes = vec![
-                       "00", 
+                       "00",
                        "fc",
                        "fd00fd",
                        "fdffff",
@@ -1286,7 +1345,7 @@ mod tests {
                        "ff0000000100000000",
                        "ffffffffffffffffff"
                ];
-               for i in 0..=7 {        
+               for i in 0..=7 {
                        let mut stream = crate::io::Cursor::new(::hex::decode(bytes[i]).unwrap());
                        assert_eq!(super::BigSize::read(&mut stream).unwrap().0, values[i]);
                        let mut stream = super::VecWriter(Vec::new());
index 92a43b3fde5282fb4420d4981c1e23b3d9bd0e81..6067bcdd438e2ea676c2978e8bdd09b103d06893 100644 (file)
@@ -39,6 +39,11 @@ macro_rules! _encode_tlv {
                        field.write($stream)?;
                }
        };
+       ($stream: expr, $type: expr, $field: expr, optional_vec) => {
+               if !$field.is_empty() {
+                       $crate::_encode_tlv!($stream, $type, $field, vec_type);
+               }
+       };
        ($stream: expr, $type: expr, $field: expr, upgradable_required) => {
                $crate::_encode_tlv!($stream, $type, $field, required);
        };
@@ -165,6 +170,11 @@ macro_rules! _get_varint_length_prefixed_tlv_length {
                        $len.0 += field_len;
                }
        };
+       ($len: expr, $type: expr, $field: expr, optional_vec) => {
+               if !$field.is_empty() {
+                       $crate::_get_varint_length_prefixed_tlv_length!($len, $type, $field, vec_type);
+               }
+       };
        ($len: expr, $type: expr, $field: expr, (option: $trait: ident $(, $read_arg: expr)?)) => {
                $crate::_get_varint_length_prefixed_tlv_length!($len, $type, $field, option);
        };
@@ -226,6 +236,9 @@ macro_rules! _check_decoded_tlv_order {
        ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, vec_type) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, optional_vec) => {{
+               // no-op
+       }};
        ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, upgradable_required) => {{
                _check_decoded_tlv_order!($last_seen_type, $typ, $type, $field, required)
        }};
@@ -271,6 +284,9 @@ macro_rules! _check_missing_tlv {
        ($last_seen_type: expr, $type: expr, $field: ident, option) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $type: expr, $field: ident, optional_vec) => {{
+               // no-op
+       }};
        ($last_seen_type: expr, $type: expr, $field: ident, upgradable_required) => {{
                _check_missing_tlv!($last_seen_type, $type, $field, required)
        }};
@@ -308,6 +324,9 @@ macro_rules! _decode_tlv {
        ($reader: expr, $field: ident, option) => {{
                $field = Some($crate::util::ser::Readable::read(&mut $reader)?);
        }};
+       ($reader: expr, $field: ident, optional_vec) => {{
+               $crate::_decode_tlv!($reader, $field, vec_type);
+       }};
        // `upgradable_required` indicates we're reading a required TLV that may have been upgraded
        // without backwards compat. We'll error if the field is missing, and return `Ok(None)` if the
        // field is present but we can no longer understand it.
@@ -675,6 +694,9 @@ macro_rules! _init_tlv_based_struct_field {
        ($field: ident, vec_type) => {
                $field.unwrap()
        };
+       ($field: ident, optional_vec) => {
+               $field.unwrap()
+       };
 }
 
 /// Initializes the variable we are going to read the TLV into.
@@ -701,6 +723,9 @@ macro_rules! _init_tlv_field_var {
        ($field: ident, option) => {
                let mut $field = None;
        };
+       ($field: ident, optional_vec) => {
+               let mut $field = Some(Vec::new());
+       };
        ($field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {
                $crate::_init_tlv_field_var!($field, option);
        };
@@ -733,7 +758,8 @@ macro_rules! _init_and_read_tlv_fields {
 /// If `$fieldty` is `required`, then `$field` is a required field that is not an [`Option`] nor a [`Vec`].
 /// If `$fieldty` is `(default_value, $default)`, then `$field` will be set to `$default` if not present.
 /// If `$fieldty` is `option`, then `$field` is optional field.
-/// If `$fieldty` is `vec_type`, then `$field` is a [`Vec`], which needs to have its individual elements serialized.
+/// If `$fieldty` is `optional_vec`, then `$field` is a [`Vec`], which needs to have its individual elements serialized.
+///    Note that for `optional_vec` no bytes are written if the vec is empty
 ///
 /// For example,
 /// ```
@@ -749,7 +775,7 @@ macro_rules! _init_and_read_tlv_fields {
 ///    (0, tlv_integer, required),
 ///    (1, tlv_default_integer, (default_value, 7)),
 ///    (2, tlv_optional_integer, option),
-///    (3, tlv_vec_type_integer, vec_type),
+///    (3, tlv_vec_type_integer, optional_vec),
 /// });
 /// ```
 ///
@@ -909,7 +935,7 @@ macro_rules! _impl_writeable_tlv_based_enum_common {
 /// ```ignore
 /// impl_writeable_tlv_based_enum!(EnumName,
 ///   (0, StructVariantA) => {(0, required_variant_field, required), (1, optional_variant_field, option)},
-///   (1, StructVariantB) => {(0, variant_field_a, required), (1, variant_field_b, required), (2, variant_vec_field, vec_type)};
+///   (1, StructVariantB) => {(0, variant_field_a, required), (1, variant_field_b, required), (2, variant_vec_field, optional_vec)};
 ///   (2, TupleVariantA), (3, TupleVariantB),
 /// );
 /// ```
index a34cb0cf323a337edc757579c761d1363948212b..67f6fd6b2888dfecd4f6acb3928302e2ce41ea79 100644 (file)
@@ -25,7 +25,7 @@ use crate::ln::msgs::LightningError;
 use crate::ln::script::ShutdownScript;
 use crate::routing::gossip::{EffectiveCapacity, NetworkGraph, NodeId};
 use crate::routing::utxo::{UtxoLookup, UtxoLookupError, UtxoResult};
-use crate::routing::router::{find_route, InFlightHtlcs, Route, RouteHop, RouteParameters, Router, ScorerAccountingForInFlightHtlcs};
+use crate::routing::router::{find_route, InFlightHtlcs, Path, Route, RouteParameters, Router, ScorerAccountingForInFlightHtlcs};
 use crate::routing::scoring::{ChannelUsage, Score};
 use crate::util::config::UserConfig;
 use crate::util::enforcing_trait_impls::{EnforcingSigner, EnforcementState};
@@ -60,6 +60,15 @@ use crate::chain::keysinterface::{InMemorySigner, Recipient, EntropySource, Node
 use std::time::{SystemTime, UNIX_EPOCH};
 use bitcoin::Sequence;
 
+pub fn pubkey(byte: u8) -> PublicKey {
+       let secp_ctx = Secp256k1::new();
+       PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
+}
+
+pub fn privkey(byte: u8) -> SecretKey {
+       SecretKey::from_slice(&[byte; 32]).unwrap()
+}
+
 pub struct TestVecWriter(pub Vec<u8>);
 impl Writer for TestVecWriter {
        fn write_all(&mut self, buf: &[u8]) -> Result<(), io::Error> {
@@ -106,7 +115,7 @@ impl<'a> Router for TestRouter<'a> {
                                let scorer = ScorerAccountingForInFlightHtlcs::new(locked_scorer, inflight_htlcs);
                                for path in &route.paths {
                                        let mut aggregate_msat = 0u64;
-                                       for (idx, hop) in path.iter().rev().enumerate() {
+                                       for (idx, hop) in path.hops.iter().rev().enumerate() {
                                                aggregate_msat += hop.fee_msat;
                                                let usage = ChannelUsage {
                                                        amount_msat: aggregate_msat,
@@ -116,11 +125,11 @@ impl<'a> Router for TestRouter<'a> {
 
                                                // Since the path is reversed, the last element in our iteration is the first
                                                // hop.
-                                               if idx == path.len() - 1 {
+                                               if idx == path.hops.len() - 1 {
                                                        scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(payer), &NodeId::from_pubkey(&hop.pubkey), usage);
                                                } else {
-                                                       let curr_hop_path_idx = path.len() - 1 - idx;
-                                                       scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(&path[curr_hop_path_idx - 1].pubkey), &NodeId::from_pubkey(&hop.pubkey), usage);
+                                                       let curr_hop_path_idx = path.hops.len() - 1 - idx;
+                                                       scorer.channel_penalty_msat(hop.short_channel_id, &NodeId::from_pubkey(&path.hops[curr_hop_path_idx - 1].pubkey), &NodeId::from_pubkey(&hop.pubkey), usage);
                                                }
                                        }
                                }
@@ -161,7 +170,7 @@ impl SignerProvider for OnlyReadsKeysInterface {
        fn derive_channel_signer(&self, _channel_value_satoshis: u64, _channel_keys_id: [u8; 32]) -> Self::Signer { unreachable!(); }
 
        fn read_chan_signer(&self, mut reader: &[u8]) -> Result<Self::Signer, msgs::DecodeError> {
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = Arc::new(Mutex::new(EnforcementState::new()));
 
                Ok(EnforcingSigner::new_with_revoked(
@@ -308,8 +317,26 @@ pub struct TestBroadcaster {
 }
 
 impl TestBroadcaster {
-       pub fn new(blocks: Arc<Mutex<Vec<(Block, u32)>>>) -> TestBroadcaster {
-               TestBroadcaster { txn_broadcasted: Mutex::new(Vec::new()), blocks }
+       pub fn new(network: Network) -> Self {
+               Self {
+                       txn_broadcasted: Mutex::new(Vec::new()),
+                       blocks: Arc::new(Mutex::new(vec![(genesis_block(network), 0)])),
+               }
+       }
+
+       pub fn with_blocks(blocks: Arc<Mutex<Vec<(Block, u32)>>>) -> Self {
+               Self { txn_broadcasted: Mutex::new(Vec::new()), blocks }
+       }
+
+       pub fn txn_broadcast(&self) -> Vec<Transaction> {
+               self.txn_broadcasted.lock().unwrap().split_off(0)
+       }
+
+       pub fn unique_txn_broadcast(&self) -> Vec<Transaction> {
+               let mut txn = self.txn_broadcasted.lock().unwrap().split_off(0);
+               let mut seen = HashSet::new();
+               txn.retain(|tx| seen.insert(tx.txid()));
+               txn
        }
 }
 
@@ -317,7 +344,7 @@ impl chaininterface::BroadcasterInterface for TestBroadcaster {
        fn broadcast_transaction(&self, tx: &Transaction) {
                let lock_time = tx.lock_time.0;
                assert!(lock_time < 1_500_000_000);
-               if lock_time > self.blocks.lock().unwrap().len() as u32 + 1 && lock_time < 500_000_000 {
+               if bitcoin::LockTime::from(tx.lock_time).is_block_height() && lock_time > self.blocks.lock().unwrap().last().unwrap().1 {
                        for inp in tx.input.iter() {
                                if inp.sequence != Sequence::MAX {
                                        panic!("We should never broadcast a transaction before its locktime ({})!", tx.lock_time);
@@ -772,7 +799,7 @@ impl SignerProvider for TestKeysInterface {
        fn read_chan_signer(&self, buffer: &[u8]) -> Result<Self::Signer, msgs::DecodeError> {
                let mut reader = io::Cursor::new(buffer);
 
-               let inner: InMemorySigner = Readable::read(&mut reader)?;
+               let inner: InMemorySigner = ReadableArgs::read(&mut reader, self)?;
                let state = self.make_enforcement_state_cell(inner.commitment_seed);
 
                Ok(EnforcingSigner::new_with_revoked(
@@ -956,13 +983,13 @@ impl Score for TestScorer {
                0
        }
 
-       fn payment_path_failed(&mut self, _actual_path: &[&RouteHop], _actual_short_channel_id: u64) {}
+       fn payment_path_failed(&mut self, _actual_path: &Path, _actual_short_channel_id: u64) {}
 
-       fn payment_path_successful(&mut self, _actual_path: &[&RouteHop]) {}
+       fn payment_path_successful(&mut self, _actual_path: &Path) {}
 
-       fn probe_failed(&mut self, _actual_path: &[&RouteHop], _: u64) {}
+       fn probe_failed(&mut self, _actual_path: &Path, _: u64) {}
 
-       fn probe_successful(&mut self, _actual_path: &[&RouteHop]) {}
+       fn probe_successful(&mut self, _actual_path: &Path) {}
 }
 
 impl Drop for TestScorer {
index 1e41b2daee509697db87bd9e67c246b5c0cd25f2..37c036da9594747ea81b142e151ff960c2892dad 100644 (file)
 
 use alloc::sync::Arc;
 use core::mem;
-use crate::sync::{Condvar, Mutex, MutexGuard};
+use crate::sync::Mutex;
 
 use crate::prelude::*;
 
-#[cfg(any(test, feature = "std"))]
-use std::time::{Duration, Instant};
+#[cfg(feature = "std")]
+use crate::sync::Condvar;
+#[cfg(feature = "std")]
+use std::time::Duration;
 
 use core::future::Future as StdFuture;
 use core::task::{Context, Poll};
@@ -30,74 +32,12 @@ use core::pin::Pin;
 /// Used to signal to one of many waiters that the condition they're waiting on has happened.
 pub(crate) struct Notifier {
        notify_pending: Mutex<(bool, Option<Arc<Mutex<FutureState>>>)>,
-       condvar: Condvar,
-}
-
-macro_rules! check_woken {
-       ($guard: expr, $retval: expr) => { {
-               if $guard.0 {
-                       $guard.0 = false;
-                       if $guard.1.as_ref().map(|l| l.lock().unwrap().complete).unwrap_or(false) {
-                               // If we're about to return as woken, and the future state is marked complete, wipe
-                               // the future state and let the next future wait until we get a new notify.
-                               $guard.1.take();
-                       }
-                       return $retval;
-               }
-       } }
 }
 
 impl Notifier {
        pub(crate) fn new() -> Self {
                Self {
                        notify_pending: Mutex::new((false, None)),
-                       condvar: Condvar::new(),
-               }
-       }
-
-       fn propagate_future_state_to_notify_flag(&self) -> MutexGuard<(bool, Option<Arc<Mutex<FutureState>>>)> {
-               let mut lock = self.notify_pending.lock().unwrap();
-               if let Some(existing_state) = &lock.1 {
-                       if existing_state.lock().unwrap().callbacks_made {
-                               // If the existing `FutureState` has completed and actually made callbacks,
-                               // consider the notification flag to have been cleared and reset the future state.
-                               lock.1.take();
-                               lock.0 = false;
-                       }
-               }
-               lock
-       }
-
-       pub(crate) fn wait(&self) {
-               loop {
-                       let mut guard = self.propagate_future_state_to_notify_flag();
-                       check_woken!(guard, ());
-                       guard = self.condvar.wait(guard).unwrap();
-                       check_woken!(guard, ());
-               }
-       }
-
-       #[cfg(any(test, feature = "std"))]
-       pub(crate) fn wait_timeout(&self, max_wait: Duration) -> bool {
-               let current_time = Instant::now();
-               loop {
-                       let mut guard = self.propagate_future_state_to_notify_flag();
-                       check_woken!(guard, true);
-                       guard = self.condvar.wait_timeout(guard, max_wait).unwrap().0;
-                       check_woken!(guard, true);
-                       // Due to spurious wakeups that can happen on `wait_timeout`, here we need to check if the
-                       // desired wait time has actually passed, and if not then restart the loop with a reduced wait
-                       // time. Note that this logic can be highly simplified through the use of
-                       // `Condvar::wait_while` and `Condvar::wait_timeout_while`, if and when our MSRV is raised to
-                       // 1.42.0.
-                       let elapsed = current_time.elapsed();
-                       if elapsed >= max_wait {
-                               return false;
-                       }
-                       match max_wait.checked_sub(elapsed) {
-                               None => return false,
-                               Some(_) => continue
-                       }
                }
        }
 
@@ -105,24 +45,31 @@ impl Notifier {
        pub(crate) fn notify(&self) {
                let mut lock = self.notify_pending.lock().unwrap();
                if let Some(future_state) = &lock.1 {
-                       if future_state.lock().unwrap().complete() {
+                       if complete_future(future_state) {
                                lock.1 = None;
                                return;
                        }
                }
                lock.0 = true;
-               mem::drop(lock);
-               self.condvar.notify_all();
        }
 
        /// Gets a [`Future`] that will get woken up with any waiters
        pub(crate) fn get_future(&self) -> Future {
-               let mut lock = self.propagate_future_state_to_notify_flag();
+               let mut lock = self.notify_pending.lock().unwrap();
+               if let Some(existing_state) = &lock.1 {
+                       if existing_state.lock().unwrap().callbacks_made {
+                               // If the existing `FutureState` has completed and actually made callbacks,
+                               // consider the notification flag to have been cleared and reset the future state.
+                               lock.1.take();
+                               lock.0 = false;
+                       }
+               }
                if let Some(existing_state) = &lock.1 {
                        Future { state: Arc::clone(&existing_state) }
                } else {
                        let state = Arc::new(Mutex::new(FutureState {
                                callbacks: Vec::new(),
+                               callbacks_with_state: Vec::new(),
                                complete: lock.0,
                                callbacks_made: false,
                        }));
@@ -137,6 +84,7 @@ impl Notifier {
        }
 }
 
+macro_rules! define_callback { ($($bounds: path),*) => {
 /// A callback which is called when a [`Future`] completes.
 ///
 /// Note that this MUST NOT call back into LDK directly, it must instead schedule actions to be
@@ -145,36 +93,50 @@ impl Notifier {
 ///
 /// Note that the [`std::future::Future`] implementation may only work for runtimes which schedule
 /// futures when they receive a wake, rather than immediately executing them.
-pub trait FutureCallback : Send {
+pub trait FutureCallback : $($bounds +)* {
        /// The method which is called.
        fn call(&self);
 }
 
-impl<F: Fn() + Send> FutureCallback for F {
+impl<F: Fn() $(+ $bounds)*> FutureCallback for F {
        fn call(&self) { (self)(); }
 }
+} }
+
+#[cfg(feature = "std")]
+define_callback!(Send);
+#[cfg(not(feature = "std"))]
+define_callback!();
 
 pub(crate) struct FutureState {
        // When we're tracking whether a callback counts as having woken the user's code, we check the
        // first bool - set to false if we're just calling a Waker, and true if we're calling an actual
        // user-provided function.
        callbacks: Vec<(bool, Box<dyn FutureCallback>)>,
+       callbacks_with_state: Vec<(bool, Box<dyn Fn(&Arc<Mutex<FutureState>>) -> () + Send>)>,
        complete: bool,
        callbacks_made: bool,
 }
 
-impl FutureState {
-       fn complete(&mut self) -> bool {
-               for (counts_as_call, callback) in self.callbacks.drain(..) {
-                       callback.call();
-                       self.callbacks_made |= counts_as_call;
-               }
-               self.complete = true;
-               self.callbacks_made
+fn complete_future(this: &Arc<Mutex<FutureState>>) -> bool {
+       let mut state_lock = this.lock().unwrap();
+       let state = &mut *state_lock;
+       for (counts_as_call, callback) in state.callbacks.drain(..) {
+               callback.call();
+               state.callbacks_made |= counts_as_call;
+       }
+       for (counts_as_call, callback) in state.callbacks_with_state.drain(..) {
+               (callback)(this);
+               state.callbacks_made |= counts_as_call;
        }
+       state.complete = true;
+       state.callbacks_made
 }
 
 /// A simple future which can complete once, and calls some callback(s) when it does so.
+///
+/// Clones can be made and all futures cloned from the same source will complete at the same time.
+#[derive(Clone)]
 pub struct Future {
        state: Arc<Mutex<FutureState>>,
 }
@@ -204,6 +166,29 @@ impl Future {
        pub fn register_callback_fn<F: 'static + FutureCallback>(&self, callback: F) {
                self.register_callback(Box::new(callback));
        }
+
+       /// Waits until this [`Future`] completes.
+       #[cfg(feature = "std")]
+       pub fn wait(self) {
+               Sleeper::from_single_future(self).wait();
+       }
+
+       /// Waits until this [`Future`] completes or the given amount of time has elapsed.
+       ///
+       /// Returns true if the [`Future`] completed, false if the time elapsed.
+       #[cfg(feature = "std")]
+       pub fn wait_timeout(self, max_wait: Duration) -> bool {
+               Sleeper::from_single_future(self).wait_timeout(max_wait)
+       }
+
+       #[cfg(test)]
+       pub fn poll_is_complete(&self) -> bool {
+               let mut state = self.state.lock().unwrap();
+               if state.complete {
+                       state.callbacks_made = true;
+                       true
+               } else { false }
+       }
 }
 
 use core::task::Waker;
@@ -229,6 +214,77 @@ impl<'a> StdFuture for Future {
        }
 }
 
+/// A struct which can be used to select across many [`Future`]s at once without relying on a full
+/// async context.
+#[cfg(feature = "std")]
+pub struct Sleeper {
+       notifiers: Vec<Arc<Mutex<FutureState>>>,
+}
+
+#[cfg(feature = "std")]
+impl Sleeper {
+       /// Constructs a new sleeper from one future, allowing blocking on it.
+       pub fn from_single_future(future: Future) -> Self {
+               Self { notifiers: vec![future.state] }
+       }
+       /// Constructs a new sleeper from two futures, allowing blocking on both at once.
+       // Note that this is the common case - a ChannelManager and ChainMonitor.
+       pub fn from_two_futures(fut_a: Future, fut_b: Future) -> Self {
+               Self { notifiers: vec![fut_a.state, fut_b.state] }
+       }
+       /// Constructs a new sleeper on many futures, allowing blocking on all at once.
+       pub fn new(futures: Vec<Future>) -> Self {
+               Self { notifiers: futures.into_iter().map(|f| f.state).collect() }
+       }
+       /// Prepares to go into a wait loop body, creating a condition variable which we can block on
+       /// and an `Arc<Mutex<Option<_>>>` which gets set to the waking `Future`'s state prior to the
+       /// condition variable being woken.
+       fn setup_wait(&self) -> (Arc<Condvar>, Arc<Mutex<Option<Arc<Mutex<FutureState>>>>>) {
+               let cv = Arc::new(Condvar::new());
+               let notified_fut_mtx = Arc::new(Mutex::new(None));
+               {
+                       for notifier_mtx in self.notifiers.iter() {
+                               let cv_ref = Arc::clone(&cv);
+                               let notified_fut_ref = Arc::clone(&notified_fut_mtx);
+                               let mut notifier = notifier_mtx.lock().unwrap();
+                               if notifier.complete {
+                                       *notified_fut_mtx.lock().unwrap() = Some(Arc::clone(&notifier_mtx));
+                                       break;
+                               }
+                               notifier.callbacks_with_state.push((false, Box::new(move |notifier_ref| {
+                                       *notified_fut_ref.lock().unwrap() = Some(Arc::clone(notifier_ref));
+                                       cv_ref.notify_all();
+                               })));
+                       }
+               }
+               (cv, notified_fut_mtx)
+       }
+
+       /// Wait until one of the [`Future`]s registered with this [`Sleeper`] has completed.
+       pub fn wait(&self) {
+               let (cv, notified_fut_mtx) = self.setup_wait();
+               let notified_fut = cv.wait_while(notified_fut_mtx.lock().unwrap(), |fut_opt| fut_opt.is_none())
+                       .unwrap().take().expect("CV wait shouldn't have returned until the notifying future was set");
+               notified_fut.lock().unwrap().callbacks_made = true;
+       }
+
+       /// Wait until one of the [`Future`]s registered with this [`Sleeper`] has completed or the
+       /// given amount of time has elapsed. Returns true if a [`Future`] completed, false if the time
+       /// elapsed.
+       pub fn wait_timeout(&self, max_wait: Duration) -> bool {
+               let (cv, notified_fut_mtx) = self.setup_wait();
+               let notified_fut =
+                       match cv.wait_timeout_while(notified_fut_mtx.lock().unwrap(), max_wait, |fut_opt| fut_opt.is_none()) {
+                               Ok((_, e)) if e.timed_out() => return false,
+                               Ok((mut notified_fut, _)) =>
+                                       notified_fut.take().expect("CV wait shouldn't have returned until the notifying future was set"),
+                               Err(_) => panic!("Previous panic while a lock was held led to a lock panic"),
+                       };
+               notified_fut.lock().unwrap().callbacks_made = true;
+               true
+       }
+}
+
 #[cfg(test)]
 mod tests {
        use super::*;
@@ -327,10 +383,7 @@ mod tests {
                let exit_thread_clone = exit_thread.clone();
                thread::spawn(move || {
                        loop {
-                               let mut lock = thread_notifier.notify_pending.lock().unwrap();
-                               lock.0 = true;
-                               thread_notifier.condvar.notify_all();
-
+                               thread_notifier.notify();
                                if exit_thread_clone.load(Ordering::SeqCst) {
                                        break
                                }
@@ -338,12 +391,12 @@ mod tests {
                });
 
                // Check that we can block indefinitely until updates are available.
-               let _ = persistence_notifier.wait();
+               let _ = persistence_notifier.get_future().wait();
 
                // Check that the Notifier will return after the given duration if updates are
                // available.
                loop {
-                       if persistence_notifier.wait_timeout(Duration::from_millis(100)) {
+                       if persistence_notifier.get_future().wait_timeout(Duration::from_millis(100)) {
                                break
                        }
                }
@@ -353,17 +406,56 @@ mod tests {
                // Check that the Notifier will return after the given duration even if no updates
                // are available.
                loop {
-                       if !persistence_notifier.wait_timeout(Duration::from_millis(100)) {
+                       if !persistence_notifier.get_future().wait_timeout(Duration::from_millis(100)) {
                                break
                        }
                }
        }
 
+       #[cfg(feature = "std")]
+       #[test]
+       fn test_state_drops() {
+               // Previously, there was a leak if a `Notifier` was `drop`ed without ever being notified
+               // but after having been slept-on. This tests for that leak.
+               use crate::sync::Arc;
+               use std::thread;
+
+               let notifier_a = Arc::new(Notifier::new());
+               let notifier_b = Arc::new(Notifier::new());
+
+               let thread_notifier_a = Arc::clone(&notifier_a);
+
+               let future_a = notifier_a.get_future();
+               let future_state_a = Arc::downgrade(&future_a.state);
+
+               let future_b = notifier_b.get_future();
+               let future_state_b = Arc::downgrade(&future_b.state);
+
+               let join_handle = thread::spawn(move || {
+                       // Let the other thread get to the wait point, then notify it.
+                       std::thread::sleep(Duration::from_millis(50));
+                       thread_notifier_a.notify();
+               });
+
+               // Wait on the other thread to finish its sleep, note that the leak only happened if we
+               // actually have to sleep here, not if we immediately return.
+               Sleeper::from_two_futures(future_a, future_b).wait();
+
+               join_handle.join().unwrap();
+
+               // then drop the notifiers and make sure the future states are gone.
+               mem::drop(notifier_a);
+               mem::drop(notifier_b);
+
+               assert!(future_state_a.upgrade().is_none() && future_state_b.upgrade().is_none());
+       }
+
        #[test]
        fn test_future_callbacks() {
                let future = Future {
                        state: Arc::new(Mutex::new(FutureState {
                                callbacks: Vec::new(),
+                               callbacks_with_state: Vec::new(),
                                complete: false,
                                callbacks_made: false,
                        }))
@@ -373,9 +465,9 @@ mod tests {
                future.register_callback(Box::new(move || assert!(!callback_ref.fetch_or(true, Ordering::SeqCst))));
 
                assert!(!callback.load(Ordering::SeqCst));
-               future.state.lock().unwrap().complete();
+               complete_future(&future.state);
                assert!(callback.load(Ordering::SeqCst));
-               future.state.lock().unwrap().complete();
+               complete_future(&future.state);
        }
 
        #[test]
@@ -383,11 +475,12 @@ mod tests {
                let future = Future {
                        state: Arc::new(Mutex::new(FutureState {
                                callbacks: Vec::new(),
+                               callbacks_with_state: Vec::new(),
                                complete: false,
                                callbacks_made: false,
                        }))
                };
-               future.state.lock().unwrap().complete();
+               complete_future(&future.state);
 
                let callback = Arc::new(AtomicBool::new(false));
                let callback_ref = Arc::clone(&callback);
@@ -421,6 +514,7 @@ mod tests {
                let mut future = Future {
                        state: Arc::new(Mutex::new(FutureState {
                                callbacks: Vec::new(),
+                               callbacks_with_state: Vec::new(),
                                complete: false,
                                callbacks_made: false,
                        }))
@@ -435,7 +529,7 @@ mod tests {
                assert_eq!(Pin::new(&mut second_future).poll(&mut Context::from_waker(&second_waker)), Poll::Pending);
                assert!(!second_woken.load(Ordering::SeqCst));
 
-               future.state.lock().unwrap().complete();
+               complete_future(&future.state);
                assert!(woken.load(Ordering::SeqCst));
                assert!(second_woken.load(Ordering::SeqCst));
                assert_eq!(Pin::new(&mut future).poll(&mut Context::from_waker(&waker)), Poll::Ready(()));
@@ -443,6 +537,7 @@ mod tests {
        }
 
        #[test]
+       #[cfg(feature = "std")]
        fn test_dropped_future_doesnt_count() {
                // Tests that if a Future gets drop'd before it is poll()ed `Ready` it doesn't count as
                // having been woken, leaving the notify-required flag set.
@@ -451,8 +546,8 @@ mod tests {
 
                // If we get a future and don't touch it we're definitely still notify-required.
                notifier.get_future();
-               assert!(notifier.wait_timeout(Duration::from_millis(1)));
-               assert!(!notifier.wait_timeout(Duration::from_millis(1)));
+               assert!(notifier.get_future().wait_timeout(Duration::from_millis(1)));
+               assert!(!notifier.get_future().wait_timeout(Duration::from_millis(1)));
 
                // Even if we poll'd once but didn't observe a `Ready`, we should be notify-required.
                let mut future = notifier.get_future();
@@ -461,7 +556,7 @@ mod tests {
 
                notifier.notify();
                assert!(woken.load(Ordering::SeqCst));
-               assert!(notifier.wait_timeout(Duration::from_millis(1)));
+               assert!(notifier.get_future().wait_timeout(Duration::from_millis(1)));
 
                // However, once we do poll `Ready` it should wipe the notify-required flag.
                let mut future = notifier.get_future();
@@ -471,7 +566,7 @@ mod tests {
                notifier.notify();
                assert!(woken.load(Ordering::SeqCst));
                assert_eq!(Pin::new(&mut future).poll(&mut Context::from_waker(&waker)), Poll::Ready(()));
-               assert!(!notifier.wait_timeout(Duration::from_millis(1)));
+               assert!(!notifier.get_future().wait_timeout(Duration::from_millis(1)));
        }
 
        #[test]
@@ -532,4 +627,67 @@ mod tests {
                assert!(woken.load(Ordering::SeqCst));
                assert_eq!(Pin::new(&mut future).poll(&mut Context::from_waker(&waker)), Poll::Ready(()));
        }
+
+       #[test]
+       #[cfg(feature = "std")]
+       fn test_multi_future_sleep() {
+               // Tests the `Sleeper` with multiple futures.
+               let notifier_a = Notifier::new();
+               let notifier_b = Notifier::new();
+
+               // Set both notifiers as woken without sleeping yet.
+               notifier_a.notify();
+               notifier_b.notify();
+               Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future()).wait();
+
+               // One future has woken us up, but the other should still have a pending notification.
+               Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future()).wait();
+
+               // However once we've slept twice, we should no longer have any pending notifications
+               assert!(!Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future())
+                       .wait_timeout(Duration::from_millis(10)));
+
+               // Test ordering somewhat more.
+               notifier_a.notify();
+               Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future()).wait();
+       }
+
+       #[test]
+       #[cfg(feature = "std")]
+       fn sleeper_with_pending_callbacks() {
+               // This is similar to the above `test_multi_future_sleep` test, but in addition registers
+               // "normal" callbacks which will cause the futures to assume notification has occurred,
+               // rather than waiting for a woken sleeper.
+               let notifier_a = Notifier::new();
+               let notifier_b = Notifier::new();
+
+               // Set both notifiers as woken without sleeping yet.
+               notifier_a.notify();
+               notifier_b.notify();
+
+               // After sleeping one future (not guaranteed which one, however) will have its notification
+               // bit cleared.
+               Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future()).wait();
+
+               // By registering a callback on the futures for both notifiers, one will complete
+               // immediately, but one will remain tied to the notifier, and will complete once the
+               // notifier is next woken, which will be considered the completion of the notification.
+               let callback_a = Arc::new(AtomicBool::new(false));
+               let callback_b = Arc::new(AtomicBool::new(false));
+               let callback_a_ref = Arc::clone(&callback_a);
+               let callback_b_ref = Arc::clone(&callback_b);
+               notifier_a.get_future().register_callback(Box::new(move || assert!(!callback_a_ref.fetch_or(true, Ordering::SeqCst))));
+               notifier_b.get_future().register_callback(Box::new(move || assert!(!callback_b_ref.fetch_or(true, Ordering::SeqCst))));
+               assert!(callback_a.load(Ordering::SeqCst) ^ callback_b.load(Ordering::SeqCst));
+
+               // If we now notify both notifiers again, the other callback will fire, completing the
+               // notification, and we'll be back to one pending notification.
+               notifier_a.notify();
+               notifier_b.notify();
+
+               assert!(callback_a.load(Ordering::SeqCst) && callback_b.load(Ordering::SeqCst));
+               Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future()).wait();
+               assert!(!Sleeper::from_two_futures(notifier_a.get_future(), notifier_b.get_future())
+                       .wait_timeout(Duration::from_millis(10)));
+       }
 }
diff --git a/pending_changelog/2059.txt b/pending_changelog/2059.txt
deleted file mode 100644 (file)
index d2ee0bc..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-## Backwards Compatibility
-
-- Providing `ChannelMonitorUpdate`s generated by LDK 0.0.115 to a
-`ChannelMonitor` on 0.0.114 or before may panic.
diff --git a/pending_changelog/2063.txt b/pending_changelog/2063.txt
deleted file mode 100644 (file)
index 8ac52a4..0000000
+++ /dev/null
@@ -1,4 +0,0 @@
-## API Updates
-
-- `Event::PaymentPathFailed::retry` will always be `None` if we initiate a payment on 0.0.115
-       then downgrade to an earlier version (#2063)