Merge pull request #1845 from TheBlueMatt/2022-11-future-wake-fix
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Fri, 11 Nov 2022 06:09:03 +0000 (06:09 +0000)
committerGitHub <noreply@github.com>
Fri, 11 Nov 2022 06:09:03 +0000 (06:09 +0000)
Fix persistence-required futures always completing instantly

56 files changed:
.github/workflows/build.yml
CONTRIBUTING.md
fuzz/src/chanmon_consistency.rs
fuzz/src/full_stack.rs
fuzz/src/onion_message.rs
lightning-background-processor/src/lib.rs
lightning-block-sync/src/init.rs
lightning-invoice/src/payment.rs
lightning-invoice/src/utils.rs
lightning-net-tokio/src/lib.rs
lightning-persister/src/lib.rs
lightning-rapid-gossip-sync/src/processing.rs
lightning/src/chain/chaininterface.rs
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/mod.rs
lightning/src/chain/onchaintx.rs
lightning/src/lib.rs
lightning/src/ln/chanmon_update_fail_tests.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/features.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/inbound_payment.rs
lightning/src/ln/monitor_tests.rs
lightning/src/ln/msgs.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/payment_tests.rs
lightning/src/ln/peer_handler.rs
lightning/src/ln/priv_short_conf_tests.rs
lightning/src/ln/reorg_tests.rs
lightning/src/ln/shutdown_tests.rs
lightning/src/offers/mod.rs [new file with mode: 0644]
lightning/src/offers/offer.rs [new file with mode: 0644]
lightning/src/onion_message/blinded_route.rs
lightning/src/onion_message/functional_tests.rs
lightning/src/onion_message/messenger.rs
lightning/src/onion_message/mod.rs
lightning/src/onion_message/packet.rs
lightning/src/routing/gossip.rs
lightning/src/routing/router.rs
lightning/src/routing/scoring.rs
lightning/src/util/chacha20poly1305rfc.rs
lightning/src/util/enforcing_trait_impls.rs
lightning/src/util/events.rs
lightning/src/util/mod.rs
lightning/src/util/persist.rs
lightning/src/util/scid_utils.rs
lightning/src/util/ser.rs
lightning/src/util/ser_macros.rs
lightning/src/util/string.rs [new file with mode: 0644]
pending_changelog/1743.txt [new file with mode: 0644]
pending_changelog/matt-idempotent-payments.txt [new file with mode: 0644]

index 171c57fcb1cf1c56f1b7a272e4a4a26a0513a673..3a67a681280187e382362aa83a2c3bbcb647799e 100644 (file)
@@ -228,7 +228,7 @@ jobs:
   benchmark:
     runs-on: ubuntu-latest
     env:
-      TOOLCHAIN: nightly
+      TOOLCHAIN: stable
     steps:
       - name: Checkout source code
         uses: actions/checkout@v3
@@ -273,7 +273,7 @@ jobs:
           cd ..
       - name: Run benchmarks on Rust ${{ matrix.toolchain }}
         run: |
-          cargo bench --features _bench_unstable
+          RUSTC_BOOTSTRAP=1 cargo bench --features _bench_unstable
 
   check_commits:
     runs-on: ubuntu-latest
index f1e4ce5171e71626f124701741e4f485a96f9ad5..6f12c21e9f9d6c9035d3225316d3e5ed42b5d3eb 100644 (file)
@@ -17,8 +17,8 @@ Communication Channels
 
 Communication about the development of LDK and `rust-lightning` happens
 primarily on the [LDK Discord](https://discord.gg/5AcknnMfBw) in the `#ldk-dev`
-channel. Additionally, live LDK devevelopment meetings are held every other
-Monday 19:00 UTC in the [LDK Dev Jitsi Meeting
+channel. Additionally, live LDK development meetings are held every other
+Monday 17:00 UTC in the [LDK Dev Jitsi Meeting
 Room](https://meet.jit.si/ldkdevmeeting). Upcoming events can be found in the
 [LDK calendar](https://calendar.google.com/calendar/embed?src=c_e6fv6vlshbpoob2mmbvblkkoj4%40group.calendar.google.com).
 
index a84dd4989322155540c9a47db376f72ac50b292d..892e4f6622bafae6cfc0bfd4c300f9bc634dd645 100644 (file)
@@ -38,7 +38,7 @@ use lightning::chain::transaction::OutPoint;
 use lightning::chain::chaininterface::{BroadcasterInterface, ConfirmationTarget, FeeEstimator};
 use lightning::chain::keysinterface::{KeyMaterial, KeysInterface, InMemorySigner, Recipient};
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-use lightning::ln::channelmanager::{self, ChainParameters, ChannelManager, PaymentSendFailure, ChannelManagerReadArgs};
+use lightning::ln::channelmanager::{self, ChainParameters, ChannelManager, PaymentSendFailure, ChannelManagerReadArgs, PaymentId};
 use lightning::ln::channel::FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE;
 use lightning::ln::msgs::{CommitmentUpdate, ChannelMessageHandler, DecodeError, UpdateAddHTLC, Init};
 use lightning::ln::script::ShutdownScript;
@@ -292,7 +292,7 @@ fn check_payment_err(send_err: PaymentSendFailure) {
        }
 }
 
-type ChanMan = ChannelManager<EnforcingSigner, Arc<TestChainMonitor>, Arc<TestBroadcaster>, Arc<KeyProvider>, Arc<FuzzEstimator>, Arc<dyn Logger>>;
+type ChanMan = ChannelManager<Arc<TestChainMonitor>, Arc<TestBroadcaster>, Arc<KeyProvider>, Arc<FuzzEstimator>, Arc<dyn Logger>>;
 
 #[inline]
 fn get_payment_secret_hash(dest: &ChanMan, payment_id: &mut u8) -> Option<(PaymentSecret, PaymentHash)> {
@@ -308,9 +308,12 @@ fn get_payment_secret_hash(dest: &ChanMan, payment_id: &mut u8) -> Option<(Payme
 }
 
 #[inline]
-fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, payment_id: &mut u8) -> bool {
+fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, payment_id: &mut u8, payment_idx: &mut u64) -> bool {
        let (payment_secret, payment_hash) =
                if let Some((secret, hash)) = get_payment_secret_hash(dest, payment_id) { (secret, hash) } else { return true; };
+       let mut payment_id = [0; 32];
+       payment_id[0..8].copy_from_slice(&payment_idx.to_ne_bytes());
+       *payment_idx += 1;
        if let Err(err) = source.send_payment(&Route {
                paths: vec![vec![RouteHop {
                        pubkey: dest.get_our_node_id(),
@@ -321,15 +324,18 @@ fn send_payment(source: &ChanMan, dest: &ChanMan, dest_chan_id: u64, amt: u64, p
                        cltv_expiry_delta: 200,
                }]],
                payment_params: None,
-       }, payment_hash, &Some(payment_secret)) {
+       }, payment_hash, &Some(payment_secret), PaymentId(payment_id)) {
                check_payment_err(err);
                false
        } else { true }
 }
 #[inline]
-fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, dest: &ChanMan, dest_chan_id: u64, amt: u64, payment_id: &mut u8) -> bool {
+fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, dest: &ChanMan, dest_chan_id: u64, amt: u64, payment_id: &mut u8, payment_idx: &mut u64) -> bool {
        let (payment_secret, payment_hash) =
                if let Some((secret, hash)) = get_payment_secret_hash(dest, payment_id) { (secret, hash) } else { return true; };
+       let mut payment_id = [0; 32];
+       payment_id[0..8].copy_from_slice(&payment_idx.to_ne_bytes());
+       *payment_idx += 1;
        if let Err(err) = source.send_payment(&Route {
                paths: vec![vec![RouteHop {
                        pubkey: middle.get_our_node_id(),
@@ -347,7 +353,7 @@ fn send_hop_payment(source: &ChanMan, middle: &ChanMan, middle_chan_id: u64, des
                        cltv_expiry_delta: 200,
                }]],
                payment_params: None,
-       }, payment_hash, &Some(payment_secret)) {
+       }, payment_hash, &Some(payment_secret), PaymentId(payment_id)) {
                check_payment_err(err);
                false
        } else { true }
@@ -553,6 +559,7 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
        let chan_b = nodes[2].list_usable_channels()[0].short_channel_id.unwrap();
 
        let mut payment_id: u8 = 0;
+       let mut payment_idx: u64 = 0;
 
        let mut chan_a_disconnected = false;
        let mut chan_b_disconnected = false;
@@ -872,6 +879,7 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
                                                        // looking like probes.
                                                },
                                                events::Event::PaymentForwarded { .. } if $node == 1 => {},
+                                               events::Event::ChannelReady { .. } => {},
                                                events::Event::PendingHTLCsForwardable { .. } => {
                                                        nodes[$node].process_pending_htlc_forwards();
                                                },
@@ -1036,61 +1044,61 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
                        },
 
                        // 1/10th the channel size:
-                       0x30 => { send_payment(&nodes[0], &nodes[1], chan_a, 10_000_000, &mut payment_id); },
-                       0x31 => { send_payment(&nodes[1], &nodes[0], chan_a, 10_000_000, &mut payment_id); },
-                       0x32 => { send_payment(&nodes[1], &nodes[2], chan_b, 10_000_000, &mut payment_id); },
-                       0x33 => { send_payment(&nodes[2], &nodes[1], chan_b, 10_000_000, &mut payment_id); },
-                       0x34 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10_000_000, &mut payment_id); },
-                       0x35 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10_000_000, &mut payment_id); },
-
-                       0x38 => { send_payment(&nodes[0], &nodes[1], chan_a, 1_000_000, &mut payment_id); },
-                       0x39 => { send_payment(&nodes[1], &nodes[0], chan_a, 1_000_000, &mut payment_id); },
-                       0x3a => { send_payment(&nodes[1], &nodes[2], chan_b, 1_000_000, &mut payment_id); },
-                       0x3b => { send_payment(&nodes[2], &nodes[1], chan_b, 1_000_000, &mut payment_id); },
-                       0x3c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1_000_000, &mut payment_id); },
-                       0x3d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1_000_000, &mut payment_id); },
-
-                       0x40 => { send_payment(&nodes[0], &nodes[1], chan_a, 100_000, &mut payment_id); },
-                       0x41 => { send_payment(&nodes[1], &nodes[0], chan_a, 100_000, &mut payment_id); },
-                       0x42 => { send_payment(&nodes[1], &nodes[2], chan_b, 100_000, &mut payment_id); },
-                       0x43 => { send_payment(&nodes[2], &nodes[1], chan_b, 100_000, &mut payment_id); },
-                       0x44 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 100_000, &mut payment_id); },
-                       0x45 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 100_000, &mut payment_id); },
-
-                       0x48 => { send_payment(&nodes[0], &nodes[1], chan_a, 10_000, &mut payment_id); },
-                       0x49 => { send_payment(&nodes[1], &nodes[0], chan_a, 10_000, &mut payment_id); },
-                       0x4a => { send_payment(&nodes[1], &nodes[2], chan_b, 10_000, &mut payment_id); },
-                       0x4b => { send_payment(&nodes[2], &nodes[1], chan_b, 10_000, &mut payment_id); },
-                       0x4c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10_000, &mut payment_id); },
-                       0x4d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10_000, &mut payment_id); },
-
-                       0x50 => { send_payment(&nodes[0], &nodes[1], chan_a, 1_000, &mut payment_id); },
-                       0x51 => { send_payment(&nodes[1], &nodes[0], chan_a, 1_000, &mut payment_id); },
-                       0x52 => { send_payment(&nodes[1], &nodes[2], chan_b, 1_000, &mut payment_id); },
-                       0x53 => { send_payment(&nodes[2], &nodes[1], chan_b, 1_000, &mut payment_id); },
-                       0x54 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1_000, &mut payment_id); },
-                       0x55 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1_000, &mut payment_id); },
-
-                       0x58 => { send_payment(&nodes[0], &nodes[1], chan_a, 100, &mut payment_id); },
-                       0x59 => { send_payment(&nodes[1], &nodes[0], chan_a, 100, &mut payment_id); },
-                       0x5a => { send_payment(&nodes[1], &nodes[2], chan_b, 100, &mut payment_id); },
-                       0x5b => { send_payment(&nodes[2], &nodes[1], chan_b, 100, &mut payment_id); },
-                       0x5c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 100, &mut payment_id); },
-                       0x5d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 100, &mut payment_id); },
-
-                       0x60 => { send_payment(&nodes[0], &nodes[1], chan_a, 10, &mut payment_id); },
-                       0x61 => { send_payment(&nodes[1], &nodes[0], chan_a, 10, &mut payment_id); },
-                       0x62 => { send_payment(&nodes[1], &nodes[2], chan_b, 10, &mut payment_id); },
-                       0x63 => { send_payment(&nodes[2], &nodes[1], chan_b, 10, &mut payment_id); },
-                       0x64 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10, &mut payment_id); },
-                       0x65 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10, &mut payment_id); },
-
-                       0x68 => { send_payment(&nodes[0], &nodes[1], chan_a, 1, &mut payment_id); },
-                       0x69 => { send_payment(&nodes[1], &nodes[0], chan_a, 1, &mut payment_id); },
-                       0x6a => { send_payment(&nodes[1], &nodes[2], chan_b, 1, &mut payment_id); },
-                       0x6b => { send_payment(&nodes[2], &nodes[1], chan_b, 1, &mut payment_id); },
-                       0x6c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1, &mut payment_id); },
-                       0x6d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1, &mut payment_id); },
+                       0x30 => { send_payment(&nodes[0], &nodes[1], chan_a, 10_000_000, &mut payment_id, &mut payment_idx); },
+                       0x31 => { send_payment(&nodes[1], &nodes[0], chan_a, 10_000_000, &mut payment_id, &mut payment_idx); },
+                       0x32 => { send_payment(&nodes[1], &nodes[2], chan_b, 10_000_000, &mut payment_id, &mut payment_idx); },
+                       0x33 => { send_payment(&nodes[2], &nodes[1], chan_b, 10_000_000, &mut payment_id, &mut payment_idx); },
+                       0x34 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10_000_000, &mut payment_id, &mut payment_idx); },
+                       0x35 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10_000_000, &mut payment_id, &mut payment_idx); },
+
+                       0x38 => { send_payment(&nodes[0], &nodes[1], chan_a, 1_000_000, &mut payment_id, &mut payment_idx); },
+                       0x39 => { send_payment(&nodes[1], &nodes[0], chan_a, 1_000_000, &mut payment_id, &mut payment_idx); },
+                       0x3a => { send_payment(&nodes[1], &nodes[2], chan_b, 1_000_000, &mut payment_id, &mut payment_idx); },
+                       0x3b => { send_payment(&nodes[2], &nodes[1], chan_b, 1_000_000, &mut payment_id, &mut payment_idx); },
+                       0x3c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1_000_000, &mut payment_id, &mut payment_idx); },
+                       0x3d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1_000_000, &mut payment_id, &mut payment_idx); },
+
+                       0x40 => { send_payment(&nodes[0], &nodes[1], chan_a, 100_000, &mut payment_id, &mut payment_idx); },
+                       0x41 => { send_payment(&nodes[1], &nodes[0], chan_a, 100_000, &mut payment_id, &mut payment_idx); },
+                       0x42 => { send_payment(&nodes[1], &nodes[2], chan_b, 100_000, &mut payment_id, &mut payment_idx); },
+                       0x43 => { send_payment(&nodes[2], &nodes[1], chan_b, 100_000, &mut payment_id, &mut payment_idx); },
+                       0x44 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 100_000, &mut payment_id, &mut payment_idx); },
+                       0x45 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 100_000, &mut payment_id, &mut payment_idx); },
+
+                       0x48 => { send_payment(&nodes[0], &nodes[1], chan_a, 10_000, &mut payment_id, &mut payment_idx); },
+                       0x49 => { send_payment(&nodes[1], &nodes[0], chan_a, 10_000, &mut payment_id, &mut payment_idx); },
+                       0x4a => { send_payment(&nodes[1], &nodes[2], chan_b, 10_000, &mut payment_id, &mut payment_idx); },
+                       0x4b => { send_payment(&nodes[2], &nodes[1], chan_b, 10_000, &mut payment_id, &mut payment_idx); },
+                       0x4c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10_000, &mut payment_id, &mut payment_idx); },
+                       0x4d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10_000, &mut payment_id, &mut payment_idx); },
+
+                       0x50 => { send_payment(&nodes[0], &nodes[1], chan_a, 1_000, &mut payment_id, &mut payment_idx); },
+                       0x51 => { send_payment(&nodes[1], &nodes[0], chan_a, 1_000, &mut payment_id, &mut payment_idx); },
+                       0x52 => { send_payment(&nodes[1], &nodes[2], chan_b, 1_000, &mut payment_id, &mut payment_idx); },
+                       0x53 => { send_payment(&nodes[2], &nodes[1], chan_b, 1_000, &mut payment_id, &mut payment_idx); },
+                       0x54 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1_000, &mut payment_id, &mut payment_idx); },
+                       0x55 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1_000, &mut payment_id, &mut payment_idx); },
+
+                       0x58 => { send_payment(&nodes[0], &nodes[1], chan_a, 100, &mut payment_id, &mut payment_idx); },
+                       0x59 => { send_payment(&nodes[1], &nodes[0], chan_a, 100, &mut payment_id, &mut payment_idx); },
+                       0x5a => { send_payment(&nodes[1], &nodes[2], chan_b, 100, &mut payment_id, &mut payment_idx); },
+                       0x5b => { send_payment(&nodes[2], &nodes[1], chan_b, 100, &mut payment_id, &mut payment_idx); },
+                       0x5c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 100, &mut payment_id, &mut payment_idx); },
+                       0x5d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 100, &mut payment_id, &mut payment_idx); },
+
+                       0x60 => { send_payment(&nodes[0], &nodes[1], chan_a, 10, &mut payment_id, &mut payment_idx); },
+                       0x61 => { send_payment(&nodes[1], &nodes[0], chan_a, 10, &mut payment_id, &mut payment_idx); },
+                       0x62 => { send_payment(&nodes[1], &nodes[2], chan_b, 10, &mut payment_id, &mut payment_idx); },
+                       0x63 => { send_payment(&nodes[2], &nodes[1], chan_b, 10, &mut payment_id, &mut payment_idx); },
+                       0x64 => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 10, &mut payment_id, &mut payment_idx); },
+                       0x65 => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 10, &mut payment_id, &mut payment_idx); },
+
+                       0x68 => { send_payment(&nodes[0], &nodes[1], chan_a, 1, &mut payment_id, &mut payment_idx); },
+                       0x69 => { send_payment(&nodes[1], &nodes[0], chan_a, 1, &mut payment_id, &mut payment_idx); },
+                       0x6a => { send_payment(&nodes[1], &nodes[2], chan_b, 1, &mut payment_id, &mut payment_idx); },
+                       0x6b => { send_payment(&nodes[2], &nodes[1], chan_b, 1, &mut payment_id, &mut payment_idx); },
+                       0x6c => { send_hop_payment(&nodes[0], &nodes[1], chan_a, &nodes[2], chan_b, 1, &mut payment_id, &mut payment_idx); },
+                       0x6d => { send_hop_payment(&nodes[2], &nodes[1], chan_b, &nodes[0], chan_a, 1, &mut payment_id, &mut payment_idx); },
 
                        0x80 => {
                                let max_feerate = last_htlc_clear_fee_a * FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE as u32;
@@ -1173,11 +1181,11 @@ pub fn do_test<Out: Output>(data: &[u8], underlying_out: Out) {
 
                                // Finally, make sure that at least one end of each channel can make a substantial payment
                                assert!(
-                                       send_payment(&nodes[0], &nodes[1], chan_a, 10_000_000, &mut payment_id) ||
-                                       send_payment(&nodes[1], &nodes[0], chan_a, 10_000_000, &mut payment_id));
+                                       send_payment(&nodes[0], &nodes[1], chan_a, 10_000_000, &mut payment_id, &mut payment_idx) ||
+                                       send_payment(&nodes[1], &nodes[0], chan_a, 10_000_000, &mut payment_id, &mut payment_idx));
                                assert!(
-                                       send_payment(&nodes[1], &nodes[2], chan_b, 10_000_000, &mut payment_id) ||
-                                       send_payment(&nodes[2], &nodes[1], chan_b, 10_000_000, &mut payment_id));
+                                       send_payment(&nodes[1], &nodes[2], chan_b, 10_000_000, &mut payment_id, &mut payment_idx) ||
+                                       send_payment(&nodes[2], &nodes[1], chan_b, 10_000_000, &mut payment_id, &mut payment_idx));
 
                                last_htlc_clear_fee_a = fee_est_a.ret_val.load(atomic::Ordering::Acquire);
                                last_htlc_clear_fee_b = fee_est_b.ret_val.load(atomic::Ordering::Acquire);
index c885fc6de2ffda2cf9b0a6d01bc72a5464630a14..eeb3728adf59dc789653a12e4c54835a5a1ea979 100644 (file)
@@ -35,7 +35,7 @@ use lightning::chain::chainmonitor;
 use lightning::chain::transaction::OutPoint;
 use lightning::chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial, KeysInterface};
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-use lightning::ln::channelmanager::{ChainParameters, ChannelManager};
+use lightning::ln::channelmanager::{ChainParameters, ChannelManager, PaymentId};
 use lightning::ln::peer_handler::{MessageHandler,PeerManager,SocketDescriptor,IgnoringMessageHandler};
 use lightning::ln::msgs::DecodeError;
 use lightning::ln::script::ShutdownScript;
@@ -163,7 +163,6 @@ impl<'a> std::hash::Hash for Peer<'a> {
 }
 
 type ChannelMan = ChannelManager<
-       EnforcingSigner,
        Arc<chainmonitor::ChainMonitor<EnforcingSigner, Arc<dyn chain::Filter>, Arc<TestBroadcaster>, Arc<FuzzEstimator>, Arc<dyn Logger>, Arc<TestPersister>>>,
        Arc<TestBroadcaster>, Arc<KeyProvider>, Arc<FuzzEstimator>, Arc<dyn Logger>>;
 type PeerMan<'a> = PeerManager<Peer<'a>, Arc<ChannelMan>, Arc<P2PGossipSync<Arc<NetworkGraph<Arc<dyn Logger>>>, Arc<dyn chain::Access>, Arc<dyn Logger>>>, IgnoringMessageHandler, Arc<dyn Logger>, IgnoringMessageHandler>;
@@ -482,7 +481,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                sha.input(&payment_hash.0[..]);
                                payment_hash.0 = Sha256::from_engine(sha).into_inner();
                                payments_sent += 1;
-                               match channelmanager.send_payment(&route, payment_hash, &None) {
+                               match channelmanager.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)) {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
@@ -510,7 +509,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                                let mut payment_secret = PaymentSecret([0; 32]);
                                payment_secret.0[0..8].copy_from_slice(&be64_to_array(payments_sent));
                                payments_sent += 1;
-                               match channelmanager.send_payment(&route, payment_hash, &Some(payment_secret)) {
+                               match channelmanager.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
@@ -797,7 +796,7 @@ mod tests {
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000000 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000000 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 11 020203e8 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
@@ -807,7 +806,7 @@ mod tests {
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4e00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // 030012 - inbound read from peer id 0 of len 18
                // 0064 03000000000000000000000000000000 - message header indicating message length 100
@@ -854,7 +853,7 @@ mod tests {
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000001 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000001 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 11 020203e8 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
@@ -864,7 +863,7 @@ mod tests {
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4e00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // - now respond to the update_fulfill_htlc+commitment_signed messages the client sent to peer 0
                // 030012 - inbound read from peer id 0 of len 18
@@ -930,7 +929,7 @@ mod tests {
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000002 00000000000b0838 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000002 00000000000b0838 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 12 02030927c0 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
@@ -940,7 +939,7 @@ mod tests {
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4b00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 5300000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // 030012 - inbound read from peer id 0 of len 18
                // 00a4 03000000000000000000000000000000 - message header indicating message length 164
@@ -971,7 +970,7 @@ mod tests {
                // - client now fails the HTLC backwards as it was unable to extract the payment preimage (CHECK 9 duplicate and CHECK 10)
 
                let logger = Arc::new(TrackingLogger { lines: Mutex::new(HashMap::new()) });
-               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4b000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
+               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005551202030927c00401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff53000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
 
                let log_entries = logger.lines.lock().unwrap();
                assert_eq!(log_entries.get(&("lightning::ln::peer_handler".to_string(), "Handling SendAcceptChannel event in peer_handler for node 030000000000000000000000000000000000000000000000000000000000000002 for channel ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679".to_string())), Some(&1)); // 1
index 5318c0c94c0d9c0b3eb0e1971f98447a9b465714..8fa6e4231be499a0e1ba989b68ed4c0c7dcc8e0c 100644 (file)
@@ -10,7 +10,7 @@ use lightning::ln::msgs::{self, DecodeError, OnionMessageHandler};
 use lightning::ln::script::ShutdownScript;
 use lightning::util::enforcing_trait_impls::EnforcingSigner;
 use lightning::util::logger::Logger;
-use lightning::util::ser::{MaybeReadableArgs, Readable, Writeable, Writer};
+use lightning::util::ser::{Readable, Writeable, Writer};
 use lightning::onion_message::{CustomOnionMessageContents, CustomOnionMessageHandler, OnionMessenger};
 
 use crate::utils::test_logger;
@@ -67,19 +67,16 @@ impl Writeable for TestCustomMessage {
        }
 }
 
-impl MaybeReadableArgs<u64> for TestCustomMessage {
-       fn read<R: io::Read>(buffer: &mut R, _message_type: u64,) -> Result<Option<Self>, DecodeError> where Self: Sized {
-               let mut buf = Vec::new();
-               buffer.read_to_end(&mut buf)?;
-               return Ok(Some(TestCustomMessage {}))
-       }
-}
-
 struct TestCustomMessageHandler {}
 
 impl CustomOnionMessageHandler for TestCustomMessageHandler {
        type CustomMessage = TestCustomMessage;
        fn handle_custom_message(&self, _msg: Self::CustomMessage) {}
+       fn read_custom_message<R: io::Read>(&self, _message_type: u64, buffer: &mut R) -> Result<Option<Self::CustomMessage>, msgs::DecodeError> {
+               let mut buf = Vec::new();
+               buffer.read_to_end(&mut buf)?;
+               return Ok(Some(TestCustomMessage {}))
+       }
 }
 
 pub struct VecWriter(pub Vec<u8>);
index 27cd0663e3291f27e385787d99769553c012de80..ece82bcaa55a2299fd51f738050f55c52926ad78 100644 (file)
@@ -17,7 +17,7 @@ extern crate lightning_rapid_gossip_sync;
 use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use lightning::chain::chainmonitor::{ChainMonitor, Persist};
-use lightning::chain::keysinterface::{Sign, KeysInterface};
+use lightning::chain::keysinterface::KeysInterface;
 use lightning::ln::channelmanager::ChannelManager;
 use lightning::ln::msgs::{ChannelMessageHandler, OnionMessageHandler, RoutingMessageHandler};
 use lightning::ln::peer_handler::{CustomMessageHandler, PeerManager, SocketDescriptor};
@@ -192,49 +192,22 @@ where
        }
 }
 
-/// Decorates an [`EventHandler`] with common functionality provided by standard [`EventHandler`]s.
-struct DecoratingEventHandler<
-       'a,
-       E: EventHandler,
-       PGS: Deref<Target = P2PGossipSync<G, A, L>>,
-       RGS: Deref<Target = RapidGossipSync<G, L>>,
-       G: Deref<Target = NetworkGraph<L>>,
-       A: Deref,
-       L: Deref,
->
-where A::Target: chain::Access, L::Target: Logger {
-       event_handler: E,
-       gossip_sync: &'a GossipSync<PGS, RGS, G, A, L>,
-}
-
-impl<
-       'a,
-       E: EventHandler,
-       PGS: Deref<Target = P2PGossipSync<G, A, L>>,
-       RGS: Deref<Target = RapidGossipSync<G, L>>,
-       G: Deref<Target = NetworkGraph<L>>,
-       A: Deref,
-       L: Deref,
-> EventHandler for DecoratingEventHandler<'a, E, PGS, RGS, G, A, L>
-where A::Target: chain::Access, L::Target: Logger {
-       fn handle_event(&self, event: &Event) {
-               if let Some(network_graph) = self.gossip_sync.network_graph() {
-                       network_graph.handle_event(event);
+fn handle_network_graph_update<L: Deref>(
+       network_graph: &NetworkGraph<L>, event: &Event
+) where L::Target: Logger {
+       if let Event::PaymentPathFailed { ref network_update, .. } = event {
+               if let Some(network_update) = network_update {
+                       network_graph.handle_network_update(&network_update);
                }
-               self.event_handler.handle_event(event);
        }
 }
 
 macro_rules! define_run_body {
-       ($persister: ident, $event_handler: ident, $chain_monitor: ident, $channel_manager: ident,
+       ($persister: ident, $chain_monitor: ident, $process_chain_monitor_events: expr,
+        $channel_manager: ident, $process_channel_manager_events: expr,
         $gossip_sync: ident, $peer_manager: ident, $logger: ident, $scorer: ident,
         $loop_exit_check: expr, $await: expr)
        => { {
-               let event_handler = DecoratingEventHandler {
-                       event_handler: $event_handler,
-                       gossip_sync: &$gossip_sync,
-               };
-
                log_trace!($logger, "Calling ChannelManager's timer_tick_occurred on startup");
                $channel_manager.timer_tick_occurred();
 
@@ -245,8 +218,8 @@ macro_rules! define_run_body {
                let mut have_pruned = false;
 
                loop {
-                       $channel_manager.process_pending_events(&event_handler);
-                       $chain_monitor.process_pending_events(&event_handler);
+                       $process_channel_manager_events;
+                       $process_chain_monitor_events;
 
                        // Note that the PeerManager::process_events may block on ChannelManager's locks,
                        // hence it comes last here. When the ChannelManager finishes whatever it's doing,
@@ -358,14 +331,13 @@ macro_rules! define_run_body {
 /// Processes background events in a future.
 ///
 /// `sleeper` should return a future which completes in the given amount of time and returns a
-/// boolean indicating whether the background processing should continue. Once `sleeper` returns a
-/// future which outputs false, the loop will exit and this function's future will complete.
+/// boolean indicating whether the background processing should exit. Once `sleeper` returns a
+/// future which outputs true, the loop will exit and this function's future will complete.
 ///
 /// See [`BackgroundProcessor::start`] for information on which actions this handles.
 #[cfg(feature = "futures")]
 pub async fn process_events_async<
        'a,
-       Signer: 'static + Sign,
        CA: 'static + Deref + Send + Sync,
        CF: 'static + Deref + Send + Sync,
        CW: 'static + Deref + Send + Sync,
@@ -379,10 +351,11 @@ pub async fn process_events_async<
        CMH: 'static + Deref + Send + Sync,
        RMH: 'static + Deref + Send + Sync,
        OMH: 'static + Deref + Send + Sync,
-       EH: 'static + EventHandler + Send,
+       EventHandlerFuture: core::future::Future<Output = ()>,
+       EventHandler: Fn(Event) -> EventHandlerFuture,
        PS: 'static + Deref + Send,
-       M: 'static + Deref<Target = ChainMonitor<Signer, CF, T, F, L, P>> + Send + Sync,
-       CM: 'static + Deref<Target = ChannelManager<Signer, CW, T, K, F, L>> + Send + Sync,
+       M: 'static + Deref<Target = ChainMonitor<<K::Target as KeysInterface>::Signer, CF, T, F, L, P>> + Send + Sync,
+       CM: 'static + Deref<Target = ChannelManager<CW, T, K, F, L>> + Send + Sync,
        PGS: 'static + Deref<Target = P2PGossipSync<G, CA, L>> + Send + Sync,
        RGS: 'static + Deref<Target = RapidGossipSync<G, L>> + Send,
        UMH: 'static + Deref + Send + Sync,
@@ -392,32 +365,44 @@ pub async fn process_events_async<
        SleepFuture: core::future::Future<Output = bool>,
        Sleeper: Fn(Duration) -> SleepFuture
 >(
-       persister: PS, event_handler: EH, chain_monitor: M, channel_manager: CM,
+       persister: PS, event_handler: EventHandler, chain_monitor: M, channel_manager: CM,
        gossip_sync: GossipSync<PGS, RGS, G, CA, L>, peer_manager: PM, logger: L, scorer: Option<S>,
        sleeper: Sleeper,
 ) -> Result<(), std::io::Error>
 where
        CA::Target: 'static + chain::Access,
        CF::Target: 'static + chain::Filter,
-       CW::Target: 'static + chain::Watch<Signer>,
+       CW::Target: 'static + chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: 'static + BroadcasterInterface,
-       K::Target: 'static + KeysInterface<Signer = Signer>,
+       K::Target: 'static + KeysInterface,
        F::Target: 'static + FeeEstimator,
        L::Target: 'static + Logger,
-       P::Target: 'static + Persist<Signer>,
+       P::Target: 'static + Persist<<K::Target as KeysInterface>::Signer>,
        CMH::Target: 'static + ChannelMessageHandler,
        OMH::Target: 'static + OnionMessageHandler,
        RMH::Target: 'static + RoutingMessageHandler,
        UMH::Target: 'static + CustomMessageHandler,
-       PS::Target: 'static + Persister<'a, Signer, CW, T, K, F, L, SC>,
+       PS::Target: 'static + Persister<'a, CW, T, K, F, L, SC>,
 {
-       let mut should_continue = true;
-       define_run_body!(persister, event_handler, chain_monitor, channel_manager,
-               gossip_sync, peer_manager, logger, scorer, should_continue, {
+       let mut should_break = true;
+       let async_event_handler = |event| {
+               let network_graph = gossip_sync.network_graph();
+               let event_handler = &event_handler;
+               async move {
+                       if let Some(network_graph) = network_graph {
+                               handle_network_graph_update(network_graph, &event)
+                       }
+                       event_handler(event).await;
+               }
+       };
+       define_run_body!(persister,
+               chain_monitor, chain_monitor.process_pending_events_async(async_event_handler).await,
+               channel_manager, channel_manager.process_pending_events_async(async_event_handler).await,
+               gossip_sync, peer_manager, logger, scorer, should_break, {
                        select_biased! {
                                _ = channel_manager.get_persistable_update_future().fuse() => true,
-                               cont = sleeper(Duration::from_millis(100)).fuse() => {
-                                       should_continue = cont;
+                               exit = sleeper(Duration::from_millis(100)).fuse() => {
+                                       should_break = exit;
                                        false
                                }
                        }
@@ -471,7 +456,6 @@ impl BackgroundProcessor {
        /// [`NetworkGraph::write`]: lightning::routing::gossip::NetworkGraph#impl-Writeable
        pub fn start<
                'a,
-               Signer: 'static + Sign,
                CA: 'static + Deref + Send + Sync,
                CF: 'static + Deref + Send + Sync,
                CW: 'static + Deref + Send + Sync,
@@ -487,8 +471,8 @@ impl BackgroundProcessor {
                RMH: 'static + Deref + Send + Sync,
                EH: 'static + EventHandler + Send,
                PS: 'static + Deref + Send,
-               M: 'static + Deref<Target = ChainMonitor<Signer, CF, T, F, L, P>> + Send + Sync,
-               CM: 'static + Deref<Target = ChannelManager<Signer, CW, T, K, F, L>> + Send + Sync,
+               M: 'static + Deref<Target = ChainMonitor<<K::Target as KeysInterface>::Signer, CF, T, F, L, P>> + Send + Sync,
+               CM: 'static + Deref<Target = ChannelManager<CW, T, K, F, L>> + Send + Sync,
                PGS: 'static + Deref<Target = P2PGossipSync<G, CA, L>> + Send + Sync,
                RGS: 'static + Deref<Target = RapidGossipSync<G, L>> + Send,
                UMH: 'static + Deref + Send + Sync,
@@ -502,22 +486,30 @@ impl BackgroundProcessor {
        where
                CA::Target: 'static + chain::Access,
                CF::Target: 'static + chain::Filter,
-               CW::Target: 'static + chain::Watch<Signer>,
+               CW::Target: 'static + chain::Watch<<K::Target as KeysInterface>::Signer>,
                T::Target: 'static + BroadcasterInterface,
-               K::Target: 'static + KeysInterface<Signer = Signer>,
+               K::Target: 'static + KeysInterface,
                F::Target: 'static + FeeEstimator,
                L::Target: 'static + Logger,
-               P::Target: 'static + Persist<Signer>,
+               P::Target: 'static + Persist<<K::Target as KeysInterface>::Signer>,
                CMH::Target: 'static + ChannelMessageHandler,
                OMH::Target: 'static + OnionMessageHandler,
                RMH::Target: 'static + RoutingMessageHandler,
                UMH::Target: 'static + CustomMessageHandler,
-               PS::Target: 'static + Persister<'a, Signer, CW, T, K, F, L, SC>,
+               PS::Target: 'static + Persister<'a, CW, T, K, F, L, SC>,
        {
                let stop_thread = Arc::new(AtomicBool::new(false));
                let stop_thread_clone = stop_thread.clone();
                let handle = thread::spawn(move || -> Result<(), std::io::Error> {
-                       define_run_body!(persister, event_handler, chain_monitor, channel_manager,
+                       let event_handler = |event| {
+                               let network_graph = gossip_sync.network_graph();
+                               if let Some(network_graph) = network_graph {
+                                       handle_network_graph_update(network_graph, &event)
+                               }
+                               event_handler.handle_event(event);
+                       };
+                       define_run_body!(persister, chain_monitor, chain_monitor.process_pending_events(&event_handler),
+                               channel_manager, channel_manager.process_pending_events(&event_handler),
                                gossip_sync, peer_manager, logger, scorer, stop_thread.load(Ordering::Acquire),
                                channel_manager.await_persistable_update_timeout(Duration::from_millis(100)))
                });
@@ -769,7 +761,7 @@ mod tests {
                        begin_open_channel!($node_a, $node_b, $channel_value);
                        let events = $node_a.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
-                       let (temporary_channel_id, tx) = handle_funding_generation_ready!(&events[0], $channel_value);
+                       let (temporary_channel_id, tx) = handle_funding_generation_ready!(events[0], $channel_value);
                        end_open_channel!($node_a, $node_b, temporary_channel_id, tx);
                        tx
                }}
@@ -786,7 +778,7 @@ mod tests {
        macro_rules! handle_funding_generation_ready {
                ($event: expr, $channel_value: expr) => {{
                        match $event {
-                               &Event::FundingGenerationReady { temporary_channel_id, channel_value_satoshis, ref output_script, user_channel_id, .. } => {
+                               Event::FundingGenerationReady { temporary_channel_id, channel_value_satoshis, ref output_script, user_channel_id, .. } => {
                                        assert_eq!(channel_value_satoshis, $channel_value);
                                        assert_eq!(user_channel_id, 42);
 
@@ -847,7 +839,7 @@ mod tests {
                // Initiate the background processors to watch each node.
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].p2p_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                macro_rules! check_persisted_data {
@@ -909,7 +901,7 @@ mod tests {
                let nodes = create_nodes(1, "test_timer_tick_called".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
@@ -932,7 +924,7 @@ mod tests {
 
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_manager_error(std::io::ErrorKind::Other, "test"));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
                match bg_processor.join() {
                        Ok(_) => panic!("Expected error persisting manager"),
@@ -949,7 +941,7 @@ mod tests {
                let nodes = create_nodes(2, "test_persist_network_graph_error".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_graph_error(std::io::ErrorKind::Other, "test"));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].p2p_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                match bg_processor.stop() {
@@ -967,7 +959,7 @@ mod tests {
                let nodes = create_nodes(2, "test_persist_scorer_error".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir).with_scorer_error(std::io::ErrorKind::Other, "test"));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(),  nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                match bg_processor.stop() {
@@ -988,9 +980,12 @@ mod tests {
 
                // Set up a background event handler for FundingGenerationReady events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
-               let event_handler = move |event: &Event| {
-                       sender.send(handle_funding_generation_ready!(event, channel_value)).unwrap();
+               let event_handler = move |event: Event| match event {
+                       Event::FundingGenerationReady { .. } => sender.send(handle_funding_generation_ready!(event, channel_value)).unwrap(),
+                       Event::ChannelReady { .. } => {},
+                       _ => panic!("Unexpected event: {:?}", event),
                };
+
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                // Open a channel and check that the FundingGenerationReady event was handled.
@@ -1014,7 +1009,12 @@ mod tests {
 
                // Set up a background event handler for SpendableOutputs events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
-               let event_handler = move |event: &Event| sender.send(event.clone()).unwrap();
+               let event_handler = move |event: Event| match event {
+                       Event::SpendableOutputs { .. } => sender.send(event.clone()).unwrap(),
+                       Event::ChannelReady { .. } => {},
+                       Event::ChannelClosed { .. } => {},
+                       _ => panic!("Unexpected event: {:?}", event),
+               };
                let persister = Arc::new(Persister::new(data_dir));
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
@@ -1022,12 +1022,12 @@ mod tests {
                nodes[0].node.force_close_broadcasting_latest_txn(&nodes[0].node.list_channels()[0].channel_id, &nodes[1].node.get_our_node_id()).unwrap();
                let commitment_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().pop().unwrap();
                confirm_transaction_depth(&mut nodes[0], &commitment_tx, BREAKDOWN_TIMEOUT as u32);
+
                let event = receiver
                        .recv_timeout(Duration::from_secs(EVENT_DEADLINE))
-                       .expect("SpendableOutputs not handled within deadline");
+                       .expect("Events not handled within deadline");
                match event {
                        Event::SpendableOutputs { .. } => {},
-                       Event::ChannelClosed { .. } => {},
                        _ => panic!("Unexpected event: {:?}", event),
                }
 
@@ -1039,7 +1039,7 @@ mod tests {
                let nodes = create_nodes(2, "test_scorer_persistence".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                loop {
@@ -1067,7 +1067,7 @@ mod tests {
                assert!(original_graph_description.contains("42: features: 0000, node_one:"));
                assert_eq!(network_graph.read_only().channels().len(), 1);
 
-               let event_handler = |_: &_| {};
+               let event_handler = |_: _| {};
                let background_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].rapid_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
 
                loop {
@@ -1120,7 +1120,7 @@ mod tests {
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = Arc::new(Persister::new(data_dir));
                let router = DefaultRouter::new(Arc::clone(&nodes[0].network_graph), Arc::clone(&nodes[0].logger), random_seed_bytes, Arc::clone(&nodes[0].scorer));
-               let invoice_payer = Arc::new(InvoicePayer::new(Arc::clone(&nodes[0].node), router, Arc::clone(&nodes[0].logger), |_: &_| {}, Retry::Attempts(2)));
+               let invoice_payer = Arc::new(InvoicePayer::new(Arc::clone(&nodes[0].node), router, Arc::clone(&nodes[0].logger), |_: _| {}, Retry::Attempts(2)));
                let event_handler = Arc::clone(&invoice_payer);
                let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].no_gossip_sync(), nodes[0].peer_manager.clone(), nodes[0].logger.clone(), Some(nodes[0].scorer.clone()));
                assert!(bg_processor.stop().is_ok());
index 7a8fada9c3c5b03663e586f137e05eb43c9b8903..893f4b5d4d854c3e44ebf64922c46f338278fd9d 100644 (file)
@@ -61,16 +61,15 @@ BlockSourceResult<ValidatedBlockHeader> where B::Target: BlockSource {
 ///
 /// async fn init_sync<
 ///    B: BlockSource,
-///    K: KeysInterface<Signer = S>,
-///    S: keysinterface::Sign,
+///    K: KeysInterface,
 ///    T: BroadcasterInterface,
 ///    F: FeeEstimator,
 ///    L: Logger,
 ///    C: chain::Filter,
-///    P: chainmonitor::Persist<S>,
+///    P: chainmonitor::Persist<K::Signer>,
 /// >(
 ///    block_source: &B,
-///    chain_monitor: &ChainMonitor<S, &C, &T, &F, &L, &P>,
+///    chain_monitor: &ChainMonitor<K::Signer, &C, &T, &F, &L, &P>,
 ///    config: UserConfig,
 ///    keys_manager: &K,
 ///    tx_broadcaster: &T,
@@ -80,7 +79,7 @@ BlockSourceResult<ValidatedBlockHeader> where B::Target: BlockSource {
 /// ) {
 ///    // Read a serialized channel monitor paired with the block hash when it was persisted.
 ///    let serialized_monitor = "...";
-///    let (monitor_block_hash, mut monitor) = <(BlockHash, ChannelMonitor<S>)>::read(
+///    let (monitor_block_hash, mut monitor) = <(BlockHash, ChannelMonitor<K::Signer>)>::read(
 ///            &mut Cursor::new(&serialized_monitor), keys_manager).unwrap();
 ///
 ///    // Read the channel manager paired with the block hash when it was persisted.
@@ -95,7 +94,7 @@ BlockSourceResult<ValidatedBlockHeader> where B::Target: BlockSource {
 ///                    config,
 ///                    vec![&mut monitor],
 ///            );
-///            <(BlockHash, ChannelManager<S, &ChainMonitor<S, &C, &T, &F, &L, &P>, &T, &K, &F, &L>)>::read(
+///            <(BlockHash, ChannelManager<&ChainMonitor<K::Signer, &C, &T, &F, &L, &P>, &T, &K, &F, &L>)>::read(
 ///                    &mut Cursor::new(&serialized_manager), read_args).unwrap()
 ///    };
 ///
index 4b2a682b3fa2142bb5b491a0010e835a5e3c5728..cbd760c37d02d045c82cbc942432c1f2c82dc419 100644 (file)
 //! # use lightning::ln::channelmanager::{ChannelDetails, PaymentId, PaymentSendFailure};
 //! # use lightning::ln::msgs::LightningError;
 //! # use lightning::routing::gossip::NodeId;
-//! # use lightning::routing::router::{Route, RouteHop, RouteParameters};
+//! # use lightning::routing::router::{InFlightHtlcs, Route, RouteHop, RouteParameters, Router};
 //! # use lightning::routing::scoring::{ChannelUsage, Score};
 //! # use lightning::util::events::{Event, EventHandler, EventsProvider};
 //! # use lightning::util::logger::{Logger, Record};
 //! # use lightning::util::ser::{Writeable, Writer};
 //! # use lightning_invoice::Invoice;
-//! # use lightning_invoice::payment::{InFlightHtlcs, InvoicePayer, Payer, Retry, Router};
+//! # use lightning_invoice::payment::{InvoicePayer, Payer, Retry, ScoringRouter};
 //! # use secp256k1::PublicKey;
 //! # use std::cell::RefCell;
 //! # use std::ops::Deref;
 //! #     fn node_id(&self) -> PublicKey { unimplemented!() }
 //! #     fn first_hops(&self) -> Vec<ChannelDetails> { unimplemented!() }
 //! #     fn send_payment(
-//! #         &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>
-//! #     ) -> Result<PaymentId, PaymentSendFailure> { unimplemented!() }
+//! #         &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+//! #         payment_id: PaymentId
+//! #     ) -> Result<(), PaymentSendFailure> { unimplemented!() }
 //! #     fn send_spontaneous_payment(
-//! #         &self, route: &Route, payment_preimage: PaymentPreimage
-//! #     ) -> Result<PaymentId, PaymentSendFailure> { unimplemented!() }
+//! #         &self, route: &Route, payment_preimage: PaymentPreimage, payment_id: PaymentId,
+//! #     ) -> Result<(), PaymentSendFailure> { unimplemented!() }
 //! #     fn retry_payment(
 //! #         &self, route: &Route, payment_id: PaymentId
 //! #     ) -> Result<(), PaymentSendFailure> { unimplemented!() }
 //! # struct FakeRouter {}
 //! # impl Router for FakeRouter {
 //! #     fn find_route(
-//! #         &self, payer: &PublicKey, params: &RouteParameters, payment_hash: &PaymentHash,
+//! #         &self, payer: &PublicKey, params: &RouteParameters,
 //! #         first_hops: Option<&[&ChannelDetails]>, _inflight_htlcs: InFlightHtlcs
 //! #     ) -> Result<Route, LightningError> { unimplemented!() }
-//! #
+//! # }
+//! # impl ScoringRouter for FakeRouter {
 //! #     fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {  unimplemented!() }
 //! #     fn notify_payment_path_successful(&self, path: &[&RouteHop]) {  unimplemented!() }
 //! #     fn notify_payment_probe_successful(&self, path: &[&RouteHop]) {  unimplemented!() }
 //! # }
 //! #
 //! # fn main() {
-//! let event_handler = |event: &Event| {
+//! let event_handler = |event: Event| {
 //!     match event {
 //!         Event::PaymentPathFailed { .. } => println!("payment failed after retries"),
 //!         Event::PaymentSent { .. } => println!("payment successful"),
@@ -140,16 +142,14 @@ use bitcoin_hashes::Hash;
 use bitcoin_hashes::sha256::Hash as Sha256;
 
 use crate::prelude::*;
-use lightning::io;
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use lightning::ln::channelmanager::{ChannelDetails, PaymentId, PaymentSendFailure};
 use lightning::ln::msgs::LightningError;
 use lightning::routing::gossip::NodeId;
-use lightning::routing::router::{PaymentParameters, Route, RouteHop, RouteParameters};
+use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, RouteParameters, Router};
 use lightning::util::errors::APIError;
 use lightning::util::events::{Event, EventHandler};
 use lightning::util::logger::Logger;
-use lightning::util::ser::Writeable;
 use crate::time_utils::Time;
 use crate::sync::Mutex;
 
@@ -157,6 +157,7 @@ use secp256k1::PublicKey;
 
 use core::fmt;
 use core::fmt::{Debug, Display, Formatter};
+use core::future::Future;
 use core::ops::Deref;
 use core::time::Duration;
 #[cfg(feature = "std")]
@@ -176,9 +177,21 @@ use crate::time_utils;
 #[cfg(feature = "no-std")]
 type ConfiguredTime = time_utils::Eternity;
 
+/// Sealed trait with a blanket implementation to allow both sync and async implementations of event
+/// handling to exist within the InvoicePayer.
+mod sealed {
+       pub trait BaseEventHandler {}
+       impl<T> BaseEventHandler for T {}
+}
+
 /// (C-not exported) generally all users should use the [`InvoicePayer`] type alias.
-pub struct InvoicePayerUsingTime<P: Deref, R: Router, L: Deref, E: EventHandler, T: Time>
-where
+pub struct InvoicePayerUsingTime<
+       P: Deref,
+       R: ScoringRouter,
+       L: Deref,
+       E: sealed::BaseEventHandler,
+       T: Time
+> where
        P::Target: Payer,
        L::Target: Logger,
 {
@@ -242,6 +255,18 @@ impl<T: Time> Display for PaymentAttempts<T> {
 }
 
 /// A trait defining behavior of an [`Invoice`] payer.
+///
+/// While the behavior of [`InvoicePayer`] provides idempotency of duplicate `send_*payment` calls
+/// with the same [`PaymentHash`], it is up to the `Payer` to provide idempotency across restarts.
+///
+/// [`ChannelManager`] provides idempotency for duplicate payments with the same [`PaymentId`].
+///
+/// In order to trivially ensure idempotency for payments, the default `Payer` implementation
+/// reuses the [`PaymentHash`] bytes as the [`PaymentId`]. Custom implementations wishing to
+/// provide payment idempotency with a different idempotency key (i.e. [`PaymentId`]) should map
+/// the [`Invoice`] or spontaneous payment target pubkey to their own idempotency key.
+///
+/// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager
 pub trait Payer {
        /// Returns the payer's node id.
        fn node_id(&self) -> PublicKey;
@@ -251,13 +276,14 @@ pub trait Payer {
 
        /// Sends a payment over the Lightning Network using the given [`Route`].
        fn send_payment(
-               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>
-       ) -> Result<PaymentId, PaymentSendFailure>;
+               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               payment_id: PaymentId
+       ) -> Result<(), PaymentSendFailure>;
 
        /// Sends a spontaneous payment over the Lightning Network using the given [`Route`].
        fn send_spontaneous_payment(
-               &self, route: &Route, payment_preimage: PaymentPreimage
-       ) -> Result<PaymentId, PaymentSendFailure>;
+               &self, route: &Route, payment_preimage: PaymentPreimage, payment_id: PaymentId
+       ) -> Result<(), PaymentSendFailure>;
 
        /// Retries a failed payment path for the [`PaymentId`] using the given [`Route`].
        fn retry_payment(&self, route: &Route, payment_id: PaymentId) -> Result<(), PaymentSendFailure>;
@@ -266,13 +292,20 @@ pub trait Payer {
        fn abandon_payment(&self, payment_id: PaymentId);
 }
 
-/// A trait defining behavior for routing an [`Invoice`] payment.
-pub trait Router {
-       /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values.
-       fn find_route(
-               &self, payer: &PublicKey, route_params: &RouteParameters, payment_hash: &PaymentHash,
-               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
-       ) -> Result<Route, LightningError>;
+/// A trait defining behavior for a [`Router`] implementation that also supports scoring channels
+/// based on payment and probe success/failure.
+///
+/// [`Router`]: lightning::routing::router::Router
+pub trait ScoringRouter: Router {
+       /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values. Includes
+       /// `PaymentHash` and `PaymentId` to be able to correlate the request with a specific payment.
+       fn find_route_with_id(
+               &self, payer: &PublicKey, route_params: &RouteParameters,
+               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs,
+               _payment_hash: PaymentHash, _payment_id: PaymentId
+       ) -> Result<Route, LightningError> {
+               self.find_route(payer, route_params, first_hops, inflight_htlcs)
+       }
        /// Lets the router know that payment through a specific path has failed.
        fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64);
        /// Lets the router know that payment through a specific path was successful.
@@ -322,7 +355,8 @@ pub enum PaymentError {
        Sending(PaymentSendFailure),
 }
 
-impl<P: Deref, R: Router, L: Deref, E: EventHandler, T: Time> InvoicePayerUsingTime<P, R, L, E, T>
+impl<P: Deref, R: ScoringRouter, L: Deref, E: sealed::BaseEventHandler, T: Time>
+       InvoicePayerUsingTime<P, R, L, E, T>
 where
        P::Target: Payer,
        L::Target: Logger,
@@ -346,36 +380,76 @@ where
 
        /// Pays the given [`Invoice`], caching it for later use in case a retry is needed.
        ///
-       /// You should ensure that the `invoice.payment_hash()` is unique and the same payment_hash has
-       /// never been paid before. Because [`InvoicePayer`] is stateless no effort is made to do so
-       /// for you.
+       /// [`Invoice::payment_hash`] is used as the [`PaymentId`], which ensures idempotency as long
+       /// as the payment is still pending. Once the payment completes or fails, you must ensure that
+       /// a second payment with the same [`PaymentHash`] is never sent.
+       ///
+       /// If you wish to use a different payment idempotency token, see
+       /// [`Self::pay_invoice_with_id`].
        pub fn pay_invoice(&self, invoice: &Invoice) -> Result<PaymentId, PaymentError> {
+               let payment_id = PaymentId(invoice.payment_hash().into_inner());
+               self.pay_invoice_with_id(invoice, payment_id).map(|()| payment_id)
+       }
+
+       /// Pays the given [`Invoice`] with a custom idempotency key, caching the invoice for later use
+       /// in case a retry is needed.
+       ///
+       /// Note that idempotency is only guaranteed as long as the payment is still pending. Once the
+       /// payment completes or fails, no idempotency guarantees are made.
+       ///
+       /// You should ensure that the [`Invoice::payment_hash`] is unique and the same [`PaymentHash`]
+       /// has never been paid before.
+       ///
+       /// See [`Self::pay_invoice`] for a variant which uses the [`PaymentHash`] for the idempotency
+       /// token.
+       pub fn pay_invoice_with_id(&self, invoice: &Invoice, payment_id: PaymentId) -> Result<(), PaymentError> {
                if invoice.amount_milli_satoshis().is_none() {
                        Err(PaymentError::Invoice("amount missing"))
                } else {
-                       self.pay_invoice_using_amount(invoice, None)
+                       self.pay_invoice_using_amount(invoice, None, payment_id)
                }
        }
 
        /// Pays the given zero-value [`Invoice`] using the given amount, caching it for later use in
        /// case a retry is needed.
        ///
-       /// You should ensure that the `invoice.payment_hash()` is unique and the same payment_hash has
-       /// never been paid before. Because [`InvoicePayer`] is stateless no effort is made to do so
-       /// for you.
+       /// [`Invoice::payment_hash`] is used as the [`PaymentId`], which ensures idempotency as long
+       /// as the payment is still pending. Once the payment completes or fails, you must ensure that
+       /// a second payment with the same [`PaymentHash`] is never sent.
+       ///
+       /// If you wish to use a different payment idempotency token, see
+       /// [`Self::pay_zero_value_invoice_with_id`].
        pub fn pay_zero_value_invoice(
                &self, invoice: &Invoice, amount_msats: u64
        ) -> Result<PaymentId, PaymentError> {
+               let payment_id = PaymentId(invoice.payment_hash().into_inner());
+               self.pay_zero_value_invoice_with_id(invoice, amount_msats, payment_id).map(|()| payment_id)
+       }
+
+       /// Pays the given zero-value [`Invoice`] using the given amount and custom idempotency key,
+       /// caching the invoice for later use in case a retry is needed.
+       ///
+       /// Note that idempotency is only guaranteed as long as the payment is still pending. Once the
+       /// payment completes or fails, no idempotency guarantees are made.
+       ///
+       /// You should ensure that the [`Invoice::payment_hash`] is unique and the same [`PaymentHash`]
+       /// has never been paid before.
+       ///
+       /// See [`Self::pay_zero_value_invoice`] for a variant which uses the [`PaymentHash`] for the
+       /// idempotency token.
+       pub fn pay_zero_value_invoice_with_id(
+               &self, invoice: &Invoice, amount_msats: u64, payment_id: PaymentId
+       ) -> Result<(), PaymentError> {
                if invoice.amount_milli_satoshis().is_some() {
                        Err(PaymentError::Invoice("amount unexpected"))
                } else {
-                       self.pay_invoice_using_amount(invoice, Some(amount_msats))
+                       self.pay_invoice_using_amount(invoice, Some(amount_msats), payment_id)
                }
        }
 
        fn pay_invoice_using_amount(
-               &self, invoice: &Invoice, amount_msats: Option<u64>
-       ) -> Result<PaymentId, PaymentError> {
+               &self, invoice: &Invoice, amount_msats: Option<u64>, payment_id: PaymentId
+       ) -> Result<(), PaymentError> {
                debug_assert!(invoice.amount_milli_satoshis().is_some() ^ amount_msats.is_some());
 
                let payment_hash = PaymentHash(invoice.payment_hash().clone().into_inner());
@@ -398,7 +472,7 @@ where
                };
 
                let send_payment = |route: &Route| {
-                       self.payer.send_payment(route, payment_hash, &payment_secret)
+                       self.payer.send_payment(route, payment_hash, &payment_secret, payment_id)
                };
 
                self.pay_internal(&route_params, payment_hash, send_payment)
@@ -408,13 +482,41 @@ where
        /// Pays `pubkey` an amount using the hash of the given preimage, caching it for later use in
        /// case a retry is needed.
        ///
-       /// You should ensure that `payment_preimage` is unique and that its `payment_hash` has never
-       /// been paid before. Because [`InvoicePayer`] is stateless no effort is made to do so for you.
+       /// The hash of the [`PaymentPreimage`] is used as the [`PaymentId`], which ensures idempotency
+       /// as long as the payment is still pending. Once the payment completes or fails, you must
+       /// ensure that a second payment with the same [`PaymentPreimage`] is never sent.
        pub fn pay_pubkey(
                &self, pubkey: PublicKey, payment_preimage: PaymentPreimage, amount_msats: u64,
                final_cltv_expiry_delta: u32
        ) -> Result<PaymentId, PaymentError> {
                let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner());
+               let payment_id = PaymentId(payment_hash.0);
+               self.do_pay_pubkey(pubkey, payment_preimage, payment_hash, payment_id, amount_msats,
+                               final_cltv_expiry_delta)
+                       .map(|()| payment_id)
+       }
+
+       /// Pays `pubkey` an amount using the hash of the given preimage and a custom idempotency key,
+       /// caching the invoice for later use in case a retry is needed.
+       ///
+       /// Note that idempotency is only guaranteed as long as the payment is still pending. Once the
+       /// payment completes or fails, no idempotency guarantees are made.
+       ///
+       /// You should ensure that the [`PaymentPreimage`] is unique and the corresponding
+       /// [`PaymentHash`] has never been paid before.
+       pub fn pay_pubkey_with_id(
+               &self, pubkey: PublicKey, payment_preimage: PaymentPreimage, payment_id: PaymentId,
+               amount_msats: u64, final_cltv_expiry_delta: u32
+       ) -> Result<(), PaymentError> {
+               let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0).into_inner());
+               self.do_pay_pubkey(pubkey, payment_preimage, payment_hash, payment_id, amount_msats,
+                               final_cltv_expiry_delta)
+       }
+
+       fn do_pay_pubkey(
+               &self, pubkey: PublicKey, payment_preimage: PaymentPreimage, payment_hash: PaymentHash,
+               payment_id: PaymentId, amount_msats: u64, final_cltv_expiry_delta: u32
+       ) -> Result<(), PaymentError> {
                match self.payment_cache.lock().unwrap().entry(payment_hash) {
                        hash_map::Entry::Occupied(_) => return Err(PaymentError::Invoice("payment pending")),
                        hash_map::Entry::Vacant(entry) => entry.insert(PaymentInfo::new()),
@@ -427,15 +529,15 @@ where
                };
 
                let send_payment = |route: &Route| {
-                       self.payer.send_spontaneous_payment(route, payment_preimage)
+                       self.payer.send_spontaneous_payment(route, payment_preimage, payment_id)
                };
                self.pay_internal(&route_params, payment_hash, send_payment)
                        .map_err(|e| { self.payment_cache.lock().unwrap().remove(&payment_hash); e })
        }
 
-       fn pay_internal<F: FnOnce(&Route) -> Result<PaymentId, PaymentSendFailure> + Copy>(
+       fn pay_internal<F: FnOnce(&Route) -> Result<(), PaymentSendFailure> + Copy>(
                &self, params: &RouteParameters, payment_hash: PaymentHash, send_payment: F,
-       ) -> Result<PaymentId, PaymentError> {
+       ) -> Result<(), PaymentError> {
                #[cfg(feature = "std")] {
                        if has_expired(params) {
                                log_trace!(self.logger, "Invoice expired prior to send for payment {}", log_bytes!(payment_hash.0));
@@ -447,16 +549,15 @@ where
                let first_hops = self.payer.first_hops();
                let inflight_htlcs = self.create_inflight_map();
                let route = self.router.find_route(
-                       &payer, &params, &payment_hash, Some(&first_hops.iter().collect::<Vec<_>>()),
-                       inflight_htlcs
+                       &payer, &params, Some(&first_hops.iter().collect::<Vec<_>>()), inflight_htlcs
                ).map_err(|e| PaymentError::Routing(e))?;
 
                match send_payment(&route) {
-                       Ok(payment_id) => {
+                       Ok(()) => {
                                for path in route.paths {
                                        self.process_path_inflight_htlcs(payment_hash, path);
                                }
-                               Ok(payment_id)
+                               Ok(())
                        },
                        Err(e) => match e {
                                PaymentSendFailure::ParameterError(_) => Err(e),
@@ -491,13 +592,13 @@ where
                                                // consider the payment sent, so return `Ok()` here, ignoring any retry
                                                // errors.
                                                let _ = self.retry_payment(payment_id, payment_hash, &retry_data);
-                                               Ok(payment_id)
+                                               Ok(())
                                        } else {
                                                // This may happen if we send a payment and some paths fail, but
                                                // only due to a temporary monitor failure or the like, implying
                                                // they're really in-flight, but we haven't sent the initial
                                                // HTLC-Add messages yet.
-                                               Ok(payment_id)
+                                               Ok(())
                                        }
                                },
                        },
@@ -552,8 +653,7 @@ where
                let inflight_htlcs = self.create_inflight_map();
 
                let route = self.router.find_route(
-                       &payer, &params, &payment_hash, Some(&first_hops.iter().collect::<Vec<_>>()),
-                       inflight_htlcs
+                       &payer, &params, Some(&first_hops.iter().collect::<Vec<_>>()), inflight_htlcs
                );
 
                if route.is_err() {
@@ -606,9 +706,8 @@ where
                self.payment_cache.lock().unwrap().remove(payment_hash);
        }
 
-       /// Given a [`PaymentHash`], this function looks up inflight path attempts in the payment_cache.
-       /// Then, it uses the path information inside the cache to construct a HashMap mapping a channel's
-       /// short channel id and direction to the amount being sent through it.
+       /// Use path information in the payment_cache to construct a HashMap mapping a channel's short
+       /// channel id and direction to the amount being sent through it.
        ///
        /// This function should be called whenever we need information about currently used up liquidity
        /// across payments.
@@ -644,7 +743,7 @@ where
                        }
                }
 
-               InFlightHtlcs(total_inflight_map)
+               InFlightHtlcs::new(total_inflight_map)
        }
 }
 
@@ -659,12 +758,15 @@ fn has_expired(route_params: &RouteParameters) -> bool {
        } else { false }
 }
 
-impl<P: Deref, R: Router, L: Deref, E: EventHandler, T: Time> EventHandler for InvoicePayerUsingTime<P, R, L, E, T>
+impl<P: Deref, R: ScoringRouter, L: Deref, E: sealed::BaseEventHandler, T: Time>
+       InvoicePayerUsingTime<P, R, L, E, T>
 where
        P::Target: Payer,
        L::Target: Logger,
 {
-       fn handle_event(&self, event: &Event) {
+       /// Returns a bool indicating whether the processed event should be forwarded to a user-provided
+       /// event handler.
+       fn handle_event_internal(&self, event: &Event) -> bool {
                match event {
                        Event::PaymentPathFailed { payment_hash, path, ..  }
                        | Event::PaymentPathSuccessful { path, payment_hash: Some(payment_hash), .. }
@@ -694,7 +796,7 @@ where
                                        self.payer.abandon_payment(payment_id.unwrap());
                                } else if self.retry_payment(payment_id.unwrap(), *payment_hash, retry.as_ref().unwrap()).is_ok() {
                                        // We retried at least somewhat, don't provide the PaymentPathFailed event to the user.
-                                       return;
+                                       return false;
                                } else {
                                        self.payer.abandon_payment(payment_id.unwrap());
                                }
@@ -729,32 +831,37 @@ where
                }
 
                // Delegate to the decorated event handler unless the payment is retried.
-               self.event_handler.handle_event(event)
+               true
        }
 }
 
-/// A map with liquidity value (in msat) keyed by a short channel id and the direction the HTLC
-/// is traveling in. The direction boolean is determined by checking if the HTLC source's public
-/// key is less than its destination. See [`InFlightHtlcs::used_liquidity_msat`] for more
-/// details.
-pub struct InFlightHtlcs(HashMap<(u64, bool), u64>);
-
-impl InFlightHtlcs {
-       /// Returns liquidity in msat given the public key of the HTLC source, target, and short channel
-       /// id.
-       pub fn used_liquidity_msat(&self, source: &NodeId, target: &NodeId, channel_scid: u64) -> Option<u64> {
-               self.0.get(&(channel_scid, source < target)).map(|v| *v)
+impl<P: Deref, R: ScoringRouter, L: Deref, E: EventHandler, T: Time>
+       EventHandler for InvoicePayerUsingTime<P, R, L, E, T>
+where
+       P::Target: Payer,
+       L::Target: Logger,
+{
+       fn handle_event(&self, event: Event) {
+               let should_forward = self.handle_event_internal(&event);
+               if should_forward {
+                       self.event_handler.handle_event(event)
+               }
        }
 }
 
-impl Writeable for InFlightHtlcs {
-       fn write<W: lightning::util::ser::Writer>(&self, writer: &mut W) -> Result<(), io::Error> { self.0.write(writer) }
-}
-
-impl lightning::util::ser::Readable for InFlightHtlcs {
-       fn read<R: io::Read>(reader: &mut R) -> Result<Self, lightning::ln::msgs::DecodeError> {
-               let infight_map: HashMap<(u64, bool), u64> = lightning::util::ser::Readable::read(reader)?;
-               Ok(Self(infight_map))
+impl<P: Deref, R: ScoringRouter, L: Deref, T: Time, F: Future, H: Fn(Event) -> F>
+       InvoicePayerUsingTime<P, R, L, H, T>
+where
+       P::Target: Payer,
+       L::Target: Logger,
+{
+       /// Intercepts events required by the [`InvoicePayer`] and forwards them to the underlying event
+       /// handler, if necessary, to handle them asynchronously.
+       pub async fn handle_event_async(&self, event: Event) {
+               let should_forward = self.handle_event_internal(&event);
+               if should_forward {
+                       (self.event_handler)(event).await;
+               }
        }
 }
 
@@ -770,7 +877,7 @@ mod tests {
        use lightning::ln::functional_test_utils::*;
        use lightning::ln::msgs::{ChannelMessageHandler, ErrorAction, LightningError};
        use lightning::routing::gossip::{EffectiveCapacity, NodeId};
-       use lightning::routing::router::{PaymentParameters, Route, RouteHop};
+       use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteHop, Router};
        use lightning::routing::scoring::{ChannelUsage, LockableScore, Score};
        use lightning::util::test_utils::TestLogger;
        use lightning::util::errors::APIError;
@@ -853,7 +960,7 @@ mod tests {
        #[test]
        fn pays_invoice_on_first_attempt() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -869,7 +976,7 @@ mod tests {
                let payment_id = Some(invoice_payer.pay_invoice(&invoice).unwrap());
                assert_eq!(*payer.attempts.borrow(), 1);
 
-               invoice_payer.handle_event(&Event::PaymentSent {
+               invoice_payer.handle_event(Event::PaymentSent {
                        payment_id, payment_preimage, payment_hash, fee_paid_msat: None
                });
                assert_eq!(*event_handled.borrow(), true);
@@ -879,7 +986,7 @@ mod tests {
        #[test]
        fn pays_invoice_on_retry() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -907,11 +1014,11 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 2);
 
-               invoice_payer.handle_event(&Event::PaymentSent {
+               invoice_payer.handle_event(Event::PaymentSent {
                        payment_id, payment_preimage, payment_hash, fee_paid_msat: None
                });
                assert_eq!(*event_handled.borrow(), true);
@@ -920,7 +1027,7 @@ mod tests {
 
        #[test]
        fn pays_invoice_on_partial_failure() {
-               let event_handler = |_: &_| { panic!() };
+               let event_handler = |_: Event| { panic!() };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -944,7 +1051,7 @@ mod tests {
        #[test]
        fn retries_payment_path_for_unknown_payment() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -970,15 +1077,15 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 1);
 
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 2);
 
-               invoice_payer.handle_event(&Event::PaymentSent {
+               invoice_payer.handle_event(Event::PaymentSent {
                        payment_id, payment_preimage, payment_hash, fee_paid_msat: None
                });
                assert_eq!(*event_handled.borrow(), true);
@@ -988,7 +1095,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_after_max_retry_counts() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1016,7 +1123,7 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 2);
 
@@ -1032,11 +1139,11 @@ mod tests {
                                final_value_msat: final_value_msat / 2, ..TestRouter::retry_for_invoice(&invoice)
                        }),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 3);
 
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 3);
        }
@@ -1045,7 +1152,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_after_max_retry_timeout() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1075,13 +1182,13 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 2);
 
                SinceEpoch::advance(Duration::from_secs(121));
 
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event.clone());
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 2);
        }
@@ -1089,7 +1196,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_with_missing_retry_params() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1114,7 +1221,7 @@ mod tests {
                        short_channel_id: None,
                        retry: None,
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 1);
        }
@@ -1124,7 +1231,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_after_expiration() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payer = TestPayer::new();
                let router = TestRouter::new(TestScorer::new());
@@ -1144,7 +1251,7 @@ mod tests {
        #[test]
        fn fails_retrying_invoice_after_expiration() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1173,7 +1280,7 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(retry_data),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 1);
        }
@@ -1181,7 +1288,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_after_retry_error() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1209,7 +1316,7 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 2);
        }
@@ -1217,7 +1324,7 @@ mod tests {
        #[test]
        fn fails_paying_invoice_after_rejected_by_payee() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1242,7 +1349,7 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), true);
                assert_eq!(*payer.attempts.borrow(), 1);
        }
@@ -1250,7 +1357,7 @@ mod tests {
        #[test]
        fn fails_repaying_invoice_with_pending_payment() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1290,7 +1397,7 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), true);
        }
 
@@ -1300,7 +1407,7 @@ mod tests {
                let router = FailingRouter {};
                let logger = TestLogger::new();
                let invoice_payer =
-                       InvoicePayer::new(&payer, router, &logger, |_: &_| {}, Retry::Attempts(0));
+                       InvoicePayer::new(&payer, router, &logger, |_: Event| {}, Retry::Attempts(0));
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1323,7 +1430,7 @@ mod tests {
                let router = TestRouter::new(TestScorer::new());
                let logger = TestLogger::new();
                let invoice_payer =
-                       InvoicePayer::new(&payer, router, &logger, |_: &_| {}, Retry::Attempts(0));
+                       InvoicePayer::new(&payer, router, &logger, |_: Event| {}, Retry::Attempts(0));
 
                match invoice_payer.pay_invoice(&invoice) {
                        Err(PaymentError::Sending(_)) => {},
@@ -1335,7 +1442,7 @@ mod tests {
        #[test]
        fn pays_zero_value_invoice_using_amount() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = zero_value_invoice(payment_preimage);
@@ -1352,7 +1459,7 @@ mod tests {
                        Some(invoice_payer.pay_zero_value_invoice(&invoice, final_value_msat).unwrap());
                assert_eq!(*payer.attempts.borrow(), 1);
 
-               invoice_payer.handle_event(&Event::PaymentSent {
+               invoice_payer.handle_event(Event::PaymentSent {
                        payment_id, payment_preimage, payment_hash, fee_paid_msat: None
                });
                assert_eq!(*event_handled.borrow(), true);
@@ -1362,7 +1469,7 @@ mod tests {
        #[test]
        fn fails_paying_zero_value_invoice_with_amount() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payer = TestPayer::new();
                let router = TestRouter::new(TestScorer::new());
@@ -1384,7 +1491,7 @@ mod tests {
        #[test]
        fn pays_pubkey_with_amount() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let pubkey = pubkey();
                let payment_preimage = PaymentPreimage([1; 32]);
@@ -1420,11 +1527,11 @@ mod tests {
                        short_channel_id: None,
                        retry: Some(retry),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                assert_eq!(*event_handled.borrow(), false);
                assert_eq!(*payer.attempts.borrow(), 2);
 
-               invoice_payer.handle_event(&Event::PaymentSent {
+               invoice_payer.handle_event(Event::PaymentSent {
                        payment_id, payment_preimage, payment_hash, fee_paid_msat: None
                });
                assert_eq!(*event_handled.borrow(), true);
@@ -1434,7 +1541,7 @@ mod tests {
        #[test]
        fn scores_failed_channel() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1466,13 +1573,13 @@ mod tests {
                        short_channel_id,
                        retry: Some(TestRouter::retry_for_invoice(&invoice)),
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
        }
 
        #[test]
        fn scores_successful_channels() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1494,17 +1601,17 @@ mod tests {
                let event = Event::PaymentPathSuccessful {
                        payment_id, payment_hash, path: route.paths[0].clone()
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
                let event = Event::PaymentPathSuccessful {
                        payment_id, payment_hash, path: route.paths[1].clone()
                };
-               invoice_payer.handle_event(&event);
+               invoice_payer.handle_event(event);
        }
 
        #[test]
        fn generates_correct_inflight_map_data() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice = invoice(payment_preimage);
@@ -1531,7 +1638,7 @@ mod tests {
                assert_eq!(inflight_map.0.get(&(3, false)).unwrap().clone(), 74);
                assert_eq!(inflight_map.0.get(&(4, false)).unwrap().clone(), 64);
 
-               invoice_payer.handle_event(&Event::PaymentPathSuccessful {
+               invoice_payer.handle_event(Event::PaymentPathSuccessful {
                        payment_id, payment_hash, path: route.paths[0].clone()
                });
 
@@ -1550,7 +1657,7 @@ mod tests {
        fn considers_inflight_htlcs_between_invoice_payments_when_path_succeeds() {
                // First, let's just send a payment through, but only make sure one of the path completes
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let payment_invoice = invoice(payment_preimage);
@@ -1584,7 +1691,7 @@ mod tests {
 
                // Succeed 1st path, leave 2nd path inflight
                let payment_id = invoice_payer.pay_invoice(&payment_invoice).unwrap();
-               invoice_payer.handle_event(&Event::PaymentPathSuccessful {
+               invoice_payer.handle_event(Event::PaymentPathSuccessful {
                        payment_id, payment_hash, path: route.paths[0].clone()
                });
 
@@ -1601,7 +1708,7 @@ mod tests {
        fn considers_inflight_htlcs_between_retries() {
                // First, let's just send a payment through, but only make sure one of the path completes
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let payment_invoice = invoice(payment_preimage);
@@ -1642,7 +1749,7 @@ mod tests {
 
                // Fail 1st path, leave 2nd path inflight
                let payment_id = Some(invoice_payer.pay_invoice(&payment_invoice).unwrap());
-               invoice_payer.handle_event(&Event::PaymentPathFailed {
+               invoice_payer.handle_event(Event::PaymentPathFailed {
                        payment_id,
                        payment_hash,
                        network_update: None,
@@ -1654,7 +1761,7 @@ mod tests {
                });
 
                // Fails again the 1st path of our retry
-               invoice_payer.handle_event(&Event::PaymentPathFailed {
+               invoice_payer.handle_event(Event::PaymentPathFailed {
                        payment_id,
                        payment_hash,
                        network_update: None,
@@ -1672,7 +1779,7 @@ mod tests {
        #[test]
        fn accounts_for_some_inflight_htlcs_sent_during_partial_failure() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice_to_pay = invoice(payment_preimage);
@@ -1703,7 +1810,7 @@ mod tests {
        #[test]
        fn accounts_for_all_inflight_htlcs_sent_during_partial_failure() {
                let event_handled = core::cell::RefCell::new(false);
-               let event_handler = |_: &_| { *event_handled.borrow_mut() = true; };
+               let event_handler = |_: Event| { *event_handled.borrow_mut() = true; };
 
                let payment_preimage = PaymentPreimage([1; 32]);
                let invoice_to_pay = invoice(payment_preimage);
@@ -1813,7 +1920,7 @@ mod tests {
 
        impl Router for TestRouter {
                fn find_route(
-                       &self, payer: &PublicKey, route_params: &RouteParameters, _payment_hash: &PaymentHash,
+                       &self, payer: &PublicKey, route_params: &RouteParameters,
                        _first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
                ) -> Result<Route, LightningError> {
                        // Simulate calling the Scorer just as you would in find_route
@@ -1844,7 +1951,9 @@ mod tests {
                                payment_params: Some(route_params.payment_params.clone()), ..Self::route_for_value(route_params.final_value_msat)
                        })
                }
+       }
 
+       impl ScoringRouter for TestRouter {
                fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
                        self.scorer.lock().payment_path_failed(path, short_channel_id);
                }
@@ -1866,12 +1975,14 @@ mod tests {
 
        impl Router for FailingRouter {
                fn find_route(
-                       &self, _payer: &PublicKey, _params: &RouteParameters, _payment_hash: &PaymentHash,
-                       _first_hops: Option<&[&ChannelDetails]>, _inflight_htlcs: InFlightHtlcs
+                       &self, _payer: &PublicKey, _params: &RouteParameters, _first_hops: Option<&[&ChannelDetails]>,
+                       _inflight_htlcs: InFlightHtlcs,
                ) -> Result<Route, LightningError> {
                        Err(LightningError { err: String::new(), action: ErrorAction::IgnoreError })
                }
+       }
 
+       impl ScoringRouter for FailingRouter {
                fn notify_payment_path_failed(&self, _path: &[&RouteHop], _short_channel_id: u64) {}
 
                fn notify_payment_path_successful(&self, _path: &[&RouteHop]) {}
@@ -2056,13 +2167,13 @@ mod tests {
                        self
                }
 
-               fn check_attempts(&self) -> Result<PaymentId, PaymentSendFailure> {
+               fn check_attempts(&self) -> Result<(), PaymentSendFailure> {
                        let mut attempts = self.attempts.borrow_mut();
                        *attempts += 1;
 
                        match self.failing_on_attempt.borrow_mut().remove(&*attempts) {
                                Some(failure) => Err(failure),
-                               None => Ok(PaymentId([1; 32])),
+                               None => Ok(())
                        }
                }
 
@@ -2100,15 +2211,15 @@ mod tests {
 
                fn send_payment(
                        &self, route: &Route, _payment_hash: PaymentHash,
-                       _payment_secret: &Option<PaymentSecret>
-               ) -> Result<PaymentId, PaymentSendFailure> {
+                       _payment_secret: &Option<PaymentSecret>, _payment_id: PaymentId,
+               ) -> Result<(), PaymentSendFailure> {
                        self.check_value_msats(Amount::ForInvoice(route.get_total_amount()));
                        self.check_attempts()
                }
 
                fn send_spontaneous_payment(
-                       &self, route: &Route, _payment_preimage: PaymentPreimage,
-               ) -> Result<PaymentId, PaymentSendFailure> {
+                       &self, route: &Route, _payment_preimage: PaymentPreimage, _payment_id: PaymentId,
+               ) -> Result<(), PaymentSendFailure> {
                        self.check_value_msats(Amount::Spontaneous(route.get_total_amount()));
                        self.check_attempts()
                }
@@ -2117,7 +2228,7 @@ mod tests {
                        &self, route: &Route, _payment_id: PaymentId
                ) -> Result<(), PaymentSendFailure> {
                        self.check_value_msats(Amount::OnRetry(route.get_total_amount()));
-                       self.check_attempts().map(|_| ())
+                       self.check_attempts()
                }
 
                fn abandon_payment(&self, _payment_id: PaymentId) { }
@@ -2128,12 +2239,13 @@ mod tests {
 
        impl Router for ManualRouter {
                fn find_route(
-                       &self, _payer: &PublicKey, _params: &RouteParameters, _payment_hash: &PaymentHash,
-                       _first_hops: Option<&[&ChannelDetails]>, _inflight_htlcs: InFlightHtlcs
+                       &self, _payer: &PublicKey, _params: &RouteParameters, _first_hops: Option<&[&ChannelDetails]>,
+                       _inflight_htlcs: InFlightHtlcs
                ) -> Result<Route, LightningError> {
                        self.0.borrow_mut().pop_front().unwrap()
                }
-
+       }
+       impl ScoringRouter for ManualRouter {
                fn notify_payment_path_failed(&self, _path: &[&RouteHop], _short_channel_id: u64) {}
 
                fn notify_payment_path_successful(&self, _path: &[&RouteHop]) {}
@@ -2195,7 +2307,7 @@ mod tests {
                route.paths[1][0].fee_msat = 50_000_000;
                router.expect_find_route(Ok(route.clone()));
 
-               let event_handler = |_: &_| { panic!(); };
+               let event_handler = |_: Event| { panic!(); };
                let invoice_payer = InvoicePayer::new(nodes[0].node, router, nodes[0].logger, event_handler, Retry::Attempts(1));
 
                assert!(invoice_payer.pay_invoice(&create_invoice_from_channelmanager_and_duration_since_epoch(
@@ -2240,7 +2352,7 @@ mod tests {
                route.paths[1][0].fee_msat = 50_000_001;
                router.expect_find_route(Ok(route.clone()));
 
-               let event_handler = |_: &_| { panic!(); };
+               let event_handler = |_: Event| { panic!(); };
                let invoice_payer = InvoicePayer::new(nodes[0].node, router, nodes[0].logger, event_handler, Retry::Attempts(1));
 
                assert!(invoice_payer.pay_invoice(&create_invoice_from_channelmanager_and_duration_since_epoch(
@@ -2317,8 +2429,8 @@ mod tests {
                route.paths.remove(1);
                router.expect_find_route(Ok(route.clone()));
 
-               let expected_events: RefCell<VecDeque<&dyn Fn(&Event)>> = RefCell::new(VecDeque::new());
-               let event_handler = |event: &Event| {
+               let expected_events: RefCell<VecDeque<&dyn Fn(Event)>> = RefCell::new(VecDeque::new());
+               let event_handler = |event: Event| {
                        let event_checker = expected_events.borrow_mut().pop_front().unwrap();
                        event_checker(event);
                };
@@ -2393,7 +2505,7 @@ mod tests {
                // `PaymentPathFailed` being passed up to the user (us, in this case). Previously, we'd
                // treated this as "HTLC complete" and dropped the retry counter, causing us to retry again
                // if the final HTLC failed.
-               expected_events.borrow_mut().push_back(&|ev: &Event| {
+               expected_events.borrow_mut().push_back(&|ev: Event| {
                        if let Event::PaymentPathFailed { payment_failed_permanently, all_paths_failed, .. } = ev {
                                assert!(!payment_failed_permanently);
                                assert!(all_paths_failed);
@@ -2411,13 +2523,13 @@ mod tests {
                nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
                commitment_signed_dance!(nodes[0], nodes[1], &bs_fail_update.commitment_signed, false, true);
 
-               expected_events.borrow_mut().push_back(&|ev: &Event| {
+               expected_events.borrow_mut().push_back(&|ev: Event| {
                        if let Event::PaymentPathFailed { payment_failed_permanently, all_paths_failed, .. } = ev {
                                assert!(!payment_failed_permanently);
                                assert!(all_paths_failed);
                        } else { panic!("Unexpected event"); }
                });
-               expected_events.borrow_mut().push_back(&|ev: &Event| {
+               expected_events.borrow_mut().push_back(&|ev: Event| {
                        if let Event::PaymentFailed { .. } = ev {
                        } else { panic!("Unexpected event"); }
                });
index 5b16da45993ac221088bc1b8fd170a2716f360bd..b6442934da78898d56594a2f90951f47c9d35b50 100644 (file)
@@ -1,14 +1,14 @@
 //! Convenient utilities to create an invoice.
 
 use crate::{CreationError, Currency, Invoice, InvoiceBuilder, SignOrCreationError};
-use crate::payment::{InFlightHtlcs, Payer, Router};
+use crate::payment::{Payer, ScoringRouter};
 
 use crate::{prelude::*, Description, InvoiceDescription, Sha256};
 use bech32::ToBase32;
 use bitcoin_hashes::{Hash, sha256};
 use lightning::chain;
 use lightning::chain::chaininterface::{BroadcasterInterface, FeeEstimator};
-use lightning::chain::keysinterface::{Recipient, KeysInterface, Sign};
+use lightning::chain::keysinterface::{Recipient, KeysInterface};
 use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use lightning::ln::channelmanager::{ChannelDetails, ChannelManager, PaymentId, PaymentSendFailure, MIN_FINAL_CLTV_EXPIRY};
 #[cfg(feature = "std")]
@@ -16,7 +16,7 @@ use lightning::ln::channelmanager::{PhantomRouteHints, MIN_CLTV_EXPIRY_DELTA};
 use lightning::ln::inbound_payment::{create, create_from_hash, ExpandedKey};
 use lightning::ln::msgs::LightningError;
 use lightning::routing::gossip::{NetworkGraph, NodeId, RoutingFees};
-use lightning::routing::router::{Route, RouteHint, RouteHintHop, RouteParameters, find_route, RouteHop};
+use lightning::routing::router::{InFlightHtlcs, Route, RouteHint, RouteHintHop, RouteParameters, find_route, RouteHop, Router};
 use lightning::routing::scoring::{ChannelUsage, LockableScore, Score};
 use lightning::util::logger::Logger;
 use secp256k1::PublicKey;
@@ -54,7 +54,7 @@ use crate::sync::Mutex;
 /// [`ChannelManager::create_inbound_payment`]: lightning::ln::channelmanager::ChannelManager::create_inbound_payment
 /// [`ChannelManager::create_inbound_payment_for_hash`]: lightning::ln::channelmanager::ChannelManager::create_inbound_payment_for_hash
 /// [`PhantomRouteHints::channels`]: lightning::ln::channelmanager::PhantomRouteHints::channels
-pub fn create_phantom_invoice<Signer: Sign, K: Deref, L: Deref>(
+pub fn create_phantom_invoice<K: Deref, L: Deref>(
        amt_msat: Option<u64>, payment_hash: Option<PaymentHash>, description: String,
        invoice_expiry_delta_secs: u32, phantom_route_hints: Vec<PhantomRouteHints>, keys_manager: K,
        logger: L, network: Currency,
@@ -65,7 +65,7 @@ where
 {
        let description = Description::new(description).map_err(SignOrCreationError::CreationError)?;
        let description = InvoiceDescription::Direct(&description,);
-       _create_phantom_invoice::<Signer, K, L>(
+       _create_phantom_invoice::<K, L>(
                amt_msat, payment_hash, description, invoice_expiry_delta_secs, phantom_route_hints,
                keys_manager, logger, network,
        )
@@ -103,7 +103,7 @@ where
 /// [`ChannelManager::create_inbound_payment`]: lightning::ln::channelmanager::ChannelManager::create_inbound_payment
 /// [`ChannelManager::create_inbound_payment_for_hash`]: lightning::ln::channelmanager::ChannelManager::create_inbound_payment_for_hash
 /// [`PhantomRouteHints::channels`]: lightning::ln::channelmanager::PhantomRouteHints::channels
-pub fn create_phantom_invoice_with_description_hash<Signer: Sign, K: Deref, L: Deref>(
+pub fn create_phantom_invoice_with_description_hash<K: Deref, L: Deref>(
        amt_msat: Option<u64>, payment_hash: Option<PaymentHash>, invoice_expiry_delta_secs: u32,
        description_hash: Sha256, phantom_route_hints: Vec<PhantomRouteHints>, keys_manager: K,
        logger: L, network: Currency
@@ -112,14 +112,14 @@ where
        K::Target: KeysInterface,
        L::Target: Logger,
 {
-       _create_phantom_invoice::<Signer, K, L>(
+       _create_phantom_invoice::<K, L>(
                amt_msat, payment_hash, InvoiceDescription::Hash(&description_hash),
                invoice_expiry_delta_secs, phantom_route_hints, keys_manager, logger, network,
        )
 }
 
 #[cfg(feature = "std")]
-fn _create_phantom_invoice<Signer: Sign, K: Deref, L: Deref>(
+fn _create_phantom_invoice<K: Deref, L: Deref>(
        amt_msat: Option<u64>, payment_hash: Option<PaymentHash>, description: InvoiceDescription,
        invoice_expiry_delta_secs: u32, phantom_route_hints: Vec<PhantomRouteHints>, keys_manager: K,
        logger: L, network: Currency,
@@ -235,14 +235,14 @@ where
 ///
 /// `invoice_expiry_delta_secs` describes the number of seconds that the invoice is valid for
 /// in excess of the current time.
-pub fn create_invoice_from_channelmanager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
-       channelmanager: &ChannelManager<Signer, M, T, K, F, L>, keys_manager: K, logger: L,
+pub fn create_invoice_from_channelmanager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
+       channelmanager: &ChannelManager<M, T, K, F, L>, keys_manager: K, logger: L,
        network: Currency, amt_msat: Option<u64>, description: String, invoice_expiry_delta_secs: u32
 ) -> Result<Invoice, SignOrCreationError<()>>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -265,15 +265,15 @@ where
 ///
 /// `invoice_expiry_delta_secs` describes the number of seconds that the invoice is valid for
 /// in excess of the current time.
-pub fn create_invoice_from_channelmanager_with_description_hash<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
-       channelmanager: &ChannelManager<Signer, M, T, K, F, L>, keys_manager: K, logger: L,
+pub fn create_invoice_from_channelmanager_with_description_hash<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
+       channelmanager: &ChannelManager<M, T, K, F, L>, keys_manager: K, logger: L,
        network: Currency, amt_msat: Option<u64>, description_hash: Sha256,
        invoice_expiry_delta_secs: u32
 ) -> Result<Invoice, SignOrCreationError<()>>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -292,15 +292,15 @@ where
 /// See [`create_invoice_from_channelmanager_with_description_hash`]
 /// This version can be used in a `no_std` environment, where [`std::time::SystemTime`] is not
 /// available and the current time is supplied by the caller.
-pub fn create_invoice_from_channelmanager_with_description_hash_and_duration_since_epoch<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
-       channelmanager: &ChannelManager<Signer, M, T, K, F, L>, keys_manager: K, logger: L,
+pub fn create_invoice_from_channelmanager_with_description_hash_and_duration_since_epoch<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
+       channelmanager: &ChannelManager<M, T, K, F, L>, keys_manager: K, logger: L,
        network: Currency, amt_msat: Option<u64>, description_hash: Sha256,
        duration_since_epoch: Duration, invoice_expiry_delta_secs: u32
 ) -> Result<Invoice, SignOrCreationError<()>>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -314,15 +314,15 @@ where
 /// See [`create_invoice_from_channelmanager`]
 /// This version can be used in a `no_std` environment, where [`std::time::SystemTime`] is not
 /// available and the current time is supplied by the caller.
-pub fn create_invoice_from_channelmanager_and_duration_since_epoch<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
-       channelmanager: &ChannelManager<Signer, M, T, K, F, L>, keys_manager: K, logger: L,
+pub fn create_invoice_from_channelmanager_and_duration_since_epoch<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
+       channelmanager: &ChannelManager<M, T, K, F, L>, keys_manager: K, logger: L,
        network: Currency, amt_msat: Option<u64>, description: String, duration_since_epoch: Duration,
        invoice_expiry_delta_secs: u32
 ) -> Result<Invoice, SignOrCreationError<()>>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -335,15 +335,15 @@ where
        )
 }
 
-fn _create_invoice_from_channelmanager_and_duration_since_epoch<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
-       channelmanager: &ChannelManager<Signer, M, T, K, F, L>, keys_manager: K, logger: L,
+fn _create_invoice_from_channelmanager_and_duration_since_epoch<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>(
+       channelmanager: &ChannelManager<M, T, K, F, L>, keys_manager: K, logger: L,
        network: Currency, amt_msat: Option<u64>, description: InvoiceDescription,
        duration_since_epoch: Duration, invoice_expiry_delta_secs: u32
 ) -> Result<Invoice, SignOrCreationError<()>>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -552,8 +552,8 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> Router for DefaultR
        S::Target: for <'a> LockableScore<'a>,
 {
        fn find_route(
-               &self, payer: &PublicKey, params: &RouteParameters, _payment_hash: &PaymentHash,
-               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
+               &self, payer: &PublicKey, params: &RouteParameters, first_hops: Option<&[&ChannelDetails]>,
+               inflight_htlcs: InFlightHtlcs
        ) -> Result<Route, LightningError> {
                let random_seed_bytes = {
                        let mut locked_random_seed_bytes = self.random_seed_bytes.lock().unwrap();
@@ -567,7 +567,12 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> Router for DefaultR
                        &random_seed_bytes
                )
        }
+}
 
+impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> ScoringRouter for DefaultRouter<G, L, S> where
+       L::Target: Logger,
+       S::Target: for <'a> LockableScore<'a>,
+{
        fn notify_payment_path_failed(&self, path: &[&RouteHop], short_channel_id: u64) {
                self.scorer.lock().payment_path_failed(path, short_channel_id);
        }
@@ -585,11 +590,11 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, S: Deref> Router for DefaultR
        }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Payer for ChannelManager<Signer, M, T, K, F, L>
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Payer for ChannelManager<M, T, K, F, L>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -602,16 +607,16 @@ where
        }
 
        fn send_payment(
-               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>
-       ) -> Result<PaymentId, PaymentSendFailure> {
-               self.send_payment(route, payment_hash, payment_secret)
+               &self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>,
+               payment_id: PaymentId
+       ) -> Result<(), PaymentSendFailure> {
+               self.send_payment(route, payment_hash, payment_secret, payment_id)
        }
 
        fn send_spontaneous_payment(
-               &self, route: &Route, payment_preimage: PaymentPreimage,
-       ) -> Result<PaymentId, PaymentSendFailure> {
-               self.send_spontaneous_payment(route, Some(payment_preimage))
-                       .map(|(_, payment_id)| payment_id)
+               &self, route: &Route, payment_preimage: PaymentPreimage, payment_id: PaymentId,
+       ) -> Result<(), PaymentSendFailure> {
+               self.send_spontaneous_payment(route, Some(payment_preimage), payment_id).map(|_| ())
        }
 
        fn retry_payment(
@@ -681,11 +686,10 @@ mod test {
        use bitcoin_hashes::sha256::Hash as Sha256;
        use lightning::chain::keysinterface::PhantomKeysManager;
        use lightning::ln::{PaymentPreimage, PaymentHash};
-       use lightning::ln::channelmanager::{self, PhantomRouteHints, MIN_FINAL_CLTV_EXPIRY};
+       use lightning::ln::channelmanager::{self, PhantomRouteHints, MIN_FINAL_CLTV_EXPIRY, PaymentId};
        use lightning::ln::functional_test_utils::*;
        use lightning::ln::msgs::ChannelMessageHandler;
        use lightning::routing::router::{PaymentParameters, RouteParameters, find_route};
-       use lightning::util::enforcing_trait_impls::EnforcingSigner;
        use lightning::util::events::{MessageSendEvent, MessageSendEventsProvider, Event};
        use lightning::util::test_utils;
        use lightning::util::config::UserConfig;
@@ -741,7 +745,7 @@ mod test {
                let payment_event = {
                        let mut payment_hash = PaymentHash([0; 32]);
                        payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]);
-                       nodes[0].node.send_payment(&route, payment_hash, &Some(invoice.payment_secret().clone())).unwrap();
+                       nodes[0].node.send_payment(&route, payment_hash, &Some(invoice.payment_secret().clone()), PaymentId(payment_hash.0)).unwrap();
                        let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                        assert_eq!(added_monitors.len(), 1);
                        added_monitors.clear();
@@ -869,6 +873,8 @@ mod test {
                get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[0].node.get_our_node_id());
                nodes[0].node.handle_channel_ready(&nodes[2].node.get_our_node_id(), &as_channel_ready);
                get_event_msg!(nodes[0], MessageSendEvent::SendChannelUpdate, nodes[2].node.get_our_node_id());
+               expect_channel_ready_event(&nodes[0], &nodes[2].node.get_our_node_id());
+               expect_channel_ready_event(&nodes[2], &nodes[0].node.get_our_node_id());
 
                // As `msgs::ChannelUpdate` was never handled for the participating node(s) of the second
                // channel, the channel will never be assigned any `counterparty.forwarding_info`.
@@ -1009,7 +1015,7 @@ mod test {
                let non_default_invoice_expiry_secs = 4200;
 
                let invoice =
-                       crate::utils::create_phantom_invoice::<EnforcingSigner, &test_utils::TestKeysInterface, &test_utils::TestLogger>(
+                       crate::utils::create_phantom_invoice::<&test_utils::TestKeysInterface, &test_utils::TestLogger>(
                                Some(payment_amt), payment_hash, "test".to_string(), non_default_invoice_expiry_secs,
                                route_hints, &nodes[1].keys_manager, &nodes[1].logger, Currency::BitcoinTestnet
                        ).unwrap();
@@ -1046,7 +1052,7 @@ mod test {
                let (payment_event, fwd_idx) = {
                        let mut payment_hash = PaymentHash([0; 32]);
                        payment_hash.0.copy_from_slice(&invoice.payment_hash().as_ref()[0..32]);
-                       nodes[0].node.send_payment(&route, payment_hash, &Some(invoice.payment_secret().clone())).unwrap();
+                       nodes[0].node.send_payment(&route, payment_hash, &Some(invoice.payment_secret().clone()), PaymentId(payment_hash.0)).unwrap();
                        let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                        assert_eq!(added_monitors.len(), 1);
                        added_monitors.clear();
@@ -1118,7 +1124,7 @@ mod test {
                        nodes[2].node.get_phantom_route_hints(),
                ];
 
-               let invoice = crate::utils::create_phantom_invoice::<EnforcingSigner, &test_utils::TestKeysInterface, &test_utils::TestLogger>(Some(payment_amt), Some(payment_hash), "test".to_string(), 3600, route_hints, &nodes[1].keys_manager, &nodes[1].logger, Currency::BitcoinTestnet).unwrap();
+               let invoice = crate::utils::create_phantom_invoice::<&test_utils::TestKeysInterface, &test_utils::TestLogger>(Some(payment_amt), Some(payment_hash), "test".to_string(), 3600, route_hints, &nodes[1].keys_manager, &nodes[1].logger, Currency::BitcoinTestnet).unwrap();
 
                let chan_0_1 = &nodes[1].node.list_usable_channels()[0];
                assert_eq!(invoice.route_hints()[0].0[0].htlc_minimum_msat, chan_0_1.inbound_htlc_minimum_msat);
@@ -1146,7 +1152,7 @@ mod test {
                let description_hash = crate::Sha256(Hash::hash("Description hash phantom invoice".as_bytes()));
                let non_default_invoice_expiry_secs = 4200;
                let invoice = crate::utils::create_phantom_invoice_with_description_hash::<
-                       EnforcingSigner, &test_utils::TestKeysInterface, &test_utils::TestLogger,
+                       &test_utils::TestKeysInterface, &test_utils::TestLogger,
                >(
                        Some(payment_amt), None, non_default_invoice_expiry_secs, description_hash,
                        route_hints, &nodes[1].keys_manager, &nodes[1].logger, Currency::BitcoinTestnet
@@ -1257,6 +1263,8 @@ mod test {
                get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
                nodes[3].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &as_channel_ready);
                get_event_msg!(nodes[3], MessageSendEvent::SendChannelUpdate, nodes[1].node.get_our_node_id());
+               expect_channel_ready_event(&nodes[1], &nodes[3].node.get_our_node_id());
+               expect_channel_ready_event(&nodes[3], &nodes[1].node.get_our_node_id());
 
                // As `msgs::ChannelUpdate` was never handled for the participating node(s) of the third
                // channel, the channel will never be assigned any `counterparty.forwarding_info`.
@@ -1461,7 +1469,7 @@ mod test {
                        .map(|route_hint| route_hint.phantom_scid)
                        .collect::<HashSet<u64>>();
 
-               let invoice = crate::utils::create_phantom_invoice::<EnforcingSigner, &test_utils::TestKeysInterface, &test_utils::TestLogger>(invoice_amt, None, "test".to_string(), 3600, phantom_route_hints, &invoice_node.keys_manager, &invoice_node.logger, Currency::BitcoinTestnet).unwrap();
+               let invoice = crate::utils::create_phantom_invoice::<&test_utils::TestKeysInterface, &test_utils::TestLogger>(invoice_amt, None, "test".to_string(), 3600, phantom_route_hints, &invoice_node.keys_manager, &invoice_node.logger, Currency::BitcoinTestnet).unwrap();
 
                let invoice_hints = invoice.private_routes();
 
index 3fbe6aab949bb6d19afb435795c6ead0cb212c8e..7a7cc4bb0992f7589c6df480aa5cc1ff93b86007 100644 (file)
@@ -43,7 +43,7 @@
 //! async fn connect_to_node(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, their_node_id: PublicKey, addr: SocketAddr) {
 //!    lightning_net_tokio::connect_outbound(peer_manager, their_node_id, addr).await;
 //!    loop {
-//!            let event_handler = |event: &Event| {
+//!            let event_handler = |event: Event| {
 //!                    // Handle the event!
 //!            };
 //!            channel_manager.await_persistable_update();
@@ -56,7 +56,7 @@
 //! async fn accept_socket(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, socket: TcpStream) {
 //!    lightning_net_tokio::setup_inbound(peer_manager, socket);
 //!    loop {
-//!            let event_handler = |event: &Event| {
+//!            let event_handler = |event: Event| {
 //!                    // Handle the event!
 //!            };
 //!            channel_manager.await_persistable_update();
index c36609351c7d4280cd4ac5a65265ad923ddc76de..a949584a6395a1b8fcf53bec5da6891a7d93f934 100644 (file)
@@ -20,7 +20,7 @@ extern crate libc;
 use bitcoin::hash_types::{BlockHash, Txid};
 use bitcoin::hashes::hex::FromHex;
 use lightning::chain::channelmonitor::ChannelMonitor;
-use lightning::chain::keysinterface::{Sign, KeysInterface};
+use lightning::chain::keysinterface::KeysInterface;
 use lightning::util::ser::{ReadableArgs, Writeable};
 use lightning::util::persist::KVStorePersister;
 use std::fs;
@@ -59,10 +59,10 @@ impl FilesystemPersister {
        }
 
        /// Read `ChannelMonitor`s from disk.
-       pub fn read_channelmonitors<Signer: Sign, K: Deref> (
+       pub fn read_channelmonitors<K: Deref> (
                &self, keys_manager: K
-       ) -> Result<Vec<(BlockHash, ChannelMonitor<Signer>)>, std::io::Error>
-               where K::Target: KeysInterface<Signer=Signer> + Sized,
+       ) -> Result<Vec<(BlockHash, ChannelMonitor<<K::Target as KeysInterface>::Signer>)>, std::io::Error>
+               where K::Target: KeysInterface + Sized,
        {
                let mut path = PathBuf::from(&self.path_to_channel_data);
                path.push("monitors");
@@ -105,7 +105,7 @@ impl FilesystemPersister {
 
                        let contents = fs::read(&file.path())?;
                        let mut buffer = Cursor::new(&contents);
-                       match <(BlockHash, ChannelMonitor<Signer>)>::read(&mut buffer, &*keys_manager) {
+                       match <(BlockHash, ChannelMonitor<<K::Target as KeysInterface>::Signer>)>::read(&mut buffer, &*keys_manager) {
                                Ok((blockhash, channel_monitor)) => {
                                        if channel_monitor.get_funding_txo().0.txid != txid.unwrap() || channel_monitor.get_funding_txo().0.index != index.unwrap() {
                                                return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "ChannelMonitor was stored in the wrong file"));
index 1a46ef587afad0ef28d90d95ead951e1d1f8f69e..3e001ec45c4be1aa7becbaa6da9f9158f3d217fa 100644 (file)
@@ -37,12 +37,9 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                let mut prefix = [0u8; 4];
                read_cursor.read_exact(&mut prefix)?;
 
-               match prefix {
-                       GOSSIP_PREFIX => {}
-                       _ => {
-                               return Err(DecodeError::UnknownVersion.into());
-                       }
-               };
+               if prefix != GOSSIP_PREFIX {
+                       return Err(DecodeError::UnknownVersion.into());
+               }
 
                let chain_hash: BlockHash = Readable::read(read_cursor)?;
                let latest_seen_timestamp: u32 = Readable::read(read_cursor)?;
@@ -75,6 +72,7 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
 
                        let node_id_1_index: BigSize = Readable::read(read_cursor)?;
                        let node_id_2_index: BigSize = Readable::read(read_cursor)?;
+
                        if max(node_id_1_index.0, node_id_2_index.0) >= node_id_count as u64 {
                                return Err(DecodeError::InvalidValue.into());
                        };
@@ -123,49 +121,43 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                        // flags are always sent in full, and hence always need updating
                        let standard_channel_flags = channel_flags & 0b_0000_0011;
 
-                       let mut synthetic_update = if channel_flags & 0b_1000_0000 == 0 {
-                               // full update, field flags will indicate deviations from the default
-                               UnsignedChannelUpdate {
-                                       chain_hash,
-                                       short_channel_id,
-                                       timestamp: backdated_timestamp,
-                                       flags: standard_channel_flags,
-                                       cltv_expiry_delta: default_cltv_expiry_delta,
-                                       htlc_minimum_msat: default_htlc_minimum_msat,
-                                       htlc_maximum_msat: default_htlc_maximum_msat,
-                                       fee_base_msat: default_fee_base_msat,
-                                       fee_proportional_millionths: default_fee_proportional_millionths,
-                                       excess_data: Vec::new(),
-                               }
-                       } else {
+                       let mut synthetic_update = UnsignedChannelUpdate {
+                               chain_hash,
+                               short_channel_id,
+                               timestamp: backdated_timestamp,
+                               flags: standard_channel_flags,
+                               cltv_expiry_delta: default_cltv_expiry_delta,
+                               htlc_minimum_msat: default_htlc_minimum_msat,
+                               htlc_maximum_msat: default_htlc_maximum_msat,
+                               fee_base_msat: default_fee_base_msat,
+                               fee_proportional_millionths: default_fee_proportional_millionths,
+                               excess_data: Vec::new(),
+                       };
+
+                       let mut skip_update_for_unknown_channel = false;
+
+                       if (channel_flags & 0b_1000_0000) != 0 {
                                // incremental update, field flags will indicate mutated values
                                let read_only_network_graph = network_graph.read_only();
-                               let channel = read_only_network_graph
+                               if let Some(channel) = read_only_network_graph
                                        .channels()
-                                       .get(&short_channel_id)
-                                       .ok_or(LightningError {
-                                               err: "Couldn't find channel for update".to_owned(),
-                                               action: ErrorAction::IgnoreError,
-                                       })?;
-
-                               let directional_info = channel
-                                       .get_directional_info(channel_flags)
-                                       .ok_or(LightningError {
-                                               err: "Couldn't find previous directional data for update".to_owned(),
-                                               action: ErrorAction::IgnoreError,
-                                       })?;
-
-                               UnsignedChannelUpdate {
-                                       chain_hash,
-                                       short_channel_id,
-                                       timestamp: backdated_timestamp,
-                                       flags: standard_channel_flags,
-                                       cltv_expiry_delta: directional_info.cltv_expiry_delta,
-                                       htlc_minimum_msat: directional_info.htlc_minimum_msat,
-                                       htlc_maximum_msat: directional_info.htlc_maximum_msat,
-                                       fee_base_msat: directional_info.fees.base_msat,
-                                       fee_proportional_millionths: directional_info.fees.proportional_millionths,
-                                       excess_data: Vec::new(),
+                                       .get(&short_channel_id) {
+
+                                       let directional_info = channel
+                                               .get_directional_info(channel_flags)
+                                               .ok_or(LightningError {
+                                                       err: "Couldn't find previous directional data for update".to_owned(),
+                                                       action: ErrorAction::IgnoreError,
+                                               })?;
+
+                                       synthetic_update.cltv_expiry_delta = directional_info.cltv_expiry_delta;
+                                       synthetic_update.htlc_minimum_msat = directional_info.htlc_minimum_msat;
+                                       synthetic_update.htlc_maximum_msat = directional_info.htlc_maximum_msat;
+                                       synthetic_update.fee_base_msat = directional_info.fees.base_msat;
+                                       synthetic_update.fee_proportional_millionths = directional_info.fees.proportional_millionths;
+
+                               } else {
+                                       skip_update_for_unknown_channel = true;
                                }
                        };
 
@@ -194,6 +186,10 @@ impl<NG: Deref<Target=NetworkGraph<L>>, L: Deref> RapidGossipSync<NG, L> where L
                                synthetic_update.htlc_maximum_msat = htlc_maximum_msat;
                        }
 
+                       if skip_update_for_unknown_channel {
+                               continue;
+                       }
+
                        match network_graph.update_channel_unsigned(&synthetic_update) {
                                Ok(_) => {},
                                Err(LightningError { action: ErrorAction::IgnoreDuplicateGossip, .. }) => {},
@@ -254,7 +250,7 @@ mod tests {
        }
 
        #[test]
-       fn incremental_only_update_fails_without_prior_announcements() {
+       fn incremental_only_update_ignores_missing_channel() {
                let incremental_update_input = vec![
                        76, 68, 75, 1, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
                        79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 229, 183, 167,
@@ -271,12 +267,7 @@ mod tests {
 
                let rapid_sync = RapidGossipSync::new(&network_graph);
                let update_result = rapid_sync.update_network_graph(&incremental_update_input[..]);
-               assert!(update_result.is_err());
-               if let Err(GraphSyncError::LightningError(lightning_error)) = update_result {
-                       assert_eq!(lightning_error.err, "Couldn't find channel for update");
-               } else {
-                       panic!("Unexpected update result: {:?}", update_result)
-               }
+               assert!(update_result.is_ok());
        }
 
        #[test]
@@ -534,4 +525,39 @@ mod tests {
                assert!(after.contains("619737530008010752"));
                assert!(after.contains("783241506229452801"));
        }
+
+       #[test]
+       pub fn update_fails_with_unknown_version() {
+               let unknown_version_input = vec![
+                       76, 68, 75, 2, 111, 226, 140, 10, 182, 241, 179, 114, 193, 166, 162, 70, 174, 99, 247,
+                       79, 147, 30, 131, 101, 225, 90, 8, 156, 104, 214, 25, 0, 0, 0, 0, 0, 97, 227, 98, 218,
+                       0, 0, 0, 4, 2, 22, 7, 207, 206, 25, 164, 197, 231, 230, 231, 56, 102, 61, 250, 251,
+                       187, 172, 38, 46, 79, 247, 108, 44, 155, 48, 219, 238, 252, 53, 192, 6, 67, 2, 36, 125,
+                       157, 176, 223, 175, 234, 116, 94, 248, 201, 225, 97, 235, 50, 47, 115, 172, 63, 136,
+                       88, 216, 115, 11, 111, 217, 114, 84, 116, 124, 231, 107, 2, 158, 1, 242, 121, 152, 106,
+                       204, 131, 186, 35, 93, 70, 216, 10, 237, 224, 183, 89, 95, 65, 3, 83, 185, 58, 138,
+                       181, 64, 187, 103, 127, 68, 50, 2, 201, 19, 17, 138, 136, 149, 185, 226, 156, 137, 175,
+                       110, 32, 237, 0, 217, 90, 31, 100, 228, 149, 46, 219, 175, 168, 77, 4, 143, 38, 128,
+                       76, 97, 0, 0, 0, 2, 0, 0, 255, 8, 153, 192, 0, 2, 27, 0, 0, 0, 1, 0, 0, 255, 2, 68,
+                       226, 0, 6, 11, 0, 1, 2, 3, 0, 0, 0, 4, 0, 40, 0, 0, 0, 0, 0, 0, 3, 232, 0, 0, 3, 232,
+                       0, 0, 0, 1, 0, 0, 0, 0, 29, 129, 25, 192, 255, 8, 153, 192, 0, 2, 27, 0, 0, 60, 0, 0,
+                       0, 0, 0, 0, 0, 1, 0, 0, 0, 100, 0, 0, 2, 224, 0, 0, 0, 0, 58, 85, 116, 216, 0, 29, 0,
+                       0, 0, 1, 0, 0, 0, 125, 0, 0, 0, 0, 58, 85, 116, 216, 255, 2, 68, 226, 0, 6, 11, 0, 1,
+                       0, 0, 1,
+               ];
+
+               let block_hash = genesis_block(Network::Bitcoin).block_hash();
+               let logger = TestLogger::new();
+               let network_graph = NetworkGraph::new(block_hash, &logger);
+               let rapid_sync = RapidGossipSync::new(&network_graph);
+               let update_result = rapid_sync.update_network_graph(&unknown_version_input[..]);
+
+               assert!(update_result.is_err());
+
+               if let Err(GraphSyncError::DecodeError(DecodeError::UnknownVersion)) = update_result {
+                       // this is the expected error type
+               } else {
+                       panic!("Unexpected update result: {:?}", update_result)
+               }
+       }
 }
index cbf609485ce1984800fa991e22d952e33ab5db56..1a446249105c8772ead4b9f1c1167d3ea160219a 100644 (file)
@@ -25,7 +25,7 @@ pub trait BroadcasterInterface {
 
 /// An enum that represents the speed at which we want a transaction to confirm used for feerate
 /// estimation.
-#[derive(Clone, Copy, PartialEq, Eq)]
+#[derive(Clone, Copy, Hash, PartialEq, Eq)]
 pub enum ConfirmationTarget {
        /// We are happy with this transaction confirming slowly when feerate drops some.
        Background,
index f5aa47d18d76c722d221c90ca6782a5cf1cf51dc..17fe69182ee81d060bc5c7c8c092608770104c84 100644 (file)
@@ -24,7 +24,7 @@
 //! servicing [`ChannelMonitor`] updates from the client.
 
 use bitcoin::blockdata::block::BlockHeader;
-use bitcoin::hash_types::Txid;
+use bitcoin::hash_types::{Txid, BlockHash};
 
 use crate::chain;
 use crate::chain::{ChannelMonitorUpdateStatus, Filter, WatchedOutput};
@@ -36,7 +36,7 @@ use crate::util::atomic_counter::AtomicCounter;
 use crate::util::logger::Logger;
 use crate::util::errors::APIError;
 use crate::util::events;
-use crate::util::events::EventHandler;
+use crate::util::events::{Event, EventHandler};
 use crate::ln::channelmanager::ChannelDetails;
 
 use crate::prelude::*;
@@ -395,6 +395,23 @@ where C::Target: chain::Filter,
                self.monitors.read().unwrap().keys().map(|outpoint| *outpoint).collect()
        }
 
+       #[cfg(not(c_bindings))]
+       /// Lists the pending updates for each [`ChannelMonitor`] (by `OutPoint` being monitored).
+       pub fn list_pending_monitor_updates(&self) -> HashMap<OutPoint, Vec<MonitorUpdateId>> {
+               self.monitors.read().unwrap().iter().map(|(outpoint, holder)| {
+                       (*outpoint, holder.pending_monitor_updates.lock().unwrap().clone())
+               }).collect()
+       }
+
+       #[cfg(c_bindings)]
+       /// Lists the pending updates for each [`ChannelMonitor`] (by `OutPoint` being monitored).
+       pub fn list_pending_monitor_updates(&self) -> Vec<(OutPoint, Vec<MonitorUpdateId>)> {
+               self.monitors.read().unwrap().iter().map(|(outpoint, holder)| {
+                       (*outpoint, holder.pending_monitor_updates.lock().unwrap().clone())
+               }).collect()
+       }
+
+
        #[cfg(test)]
        pub fn remove_monitor(&self, funding_txo: &OutPoint) -> ChannelMonitor<ChannelSigner> {
                self.monitors.write().unwrap().remove(funding_txo).unwrap().monitor
@@ -475,10 +492,28 @@ where C::Target: chain::Filter,
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                use crate::util::events::EventsProvider;
                let events = core::cell::RefCell::new(Vec::new());
-               let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
+               let event_handler = |event: events::Event| events.borrow_mut().push(event);
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
+
+       /// Processes any events asynchronously in the order they were generated since the last call
+       /// using the given event handler.
+       ///
+       /// See the trait-level documentation of [`EventsProvider`] for requirements.
+       ///
+       /// [`EventsProvider`]: crate::util::events::EventsProvider
+       pub async fn process_pending_events_async<Future: core::future::Future, H: Fn(Event) -> Future>(
+               &self, handler: H
+       ) {
+               let mut pending_events = Vec::new();
+               for monitor_state in self.monitors.read().unwrap().values() {
+                       pending_events.append(&mut monitor_state.monitor.get_and_clear_pending_events());
+               }
+               for event in pending_events {
+                       handler(event).await;
+               }
+       }
 }
 
 impl<ChannelSigner: Sign, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref>
@@ -544,7 +579,7 @@ where
                });
        }
 
-       fn get_relevant_txids(&self) -> Vec<Txid> {
+       fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let mut txids = Vec::new();
                let monitor_states = self.monitors.read().unwrap();
                for monitor_state in monitor_states.values() {
@@ -719,8 +754,8 @@ impl<ChannelSigner: Sign, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref> even
                for monitor_state in self.monitors.read().unwrap().values() {
                        pending_events.append(&mut monitor_state.monitor.get_and_clear_pending_events());
                }
-               for event in pending_events.drain(..) {
-                       handler.handle_event(&event);
+               for event in pending_events {
+                       handler.handle_event(event);
                }
        }
        #[cfg(anchors)]
@@ -742,8 +777,8 @@ impl<ChannelSigner: Sign, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref> even
                for monitor_state in self.monitors.read().unwrap().values() {
                        pending_events.append(&mut monitor_state.monitor.get_and_clear_pending_events());
                }
-               for event in pending_events.drain(..) {
-                       handler.handle_event(&event);
+               for event in pending_events {
+                       handler.handle_event(event);
                }
        }
 }
@@ -757,7 +792,7 @@ mod tests {
        use crate::{get_htlc_update_msgs, get_local_commitment_txn, get_revoke_commit_msgs, get_route_and_payment_hash, unwrap_send_err};
        use crate::chain::{ChannelMonitorUpdateStatus, Confirm, Watch};
        use crate::chain::channelmonitor::LATENCY_GRACE_PERIOD_BLOCKS;
-       use crate::ln::channelmanager::{self, PaymentSendFailure};
+       use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs::ChannelMessageHandler;
        use crate::util::errors::APIError;
@@ -798,7 +833,22 @@ mod tests {
                // Note that updates is a HashMap so the ordering here is actually random. This shouldn't
                // fail either way but if it fails intermittently it's depending on the ordering of updates.
                let mut update_iter = updates.iter();
-               nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, update_iter.next().unwrap().clone()).unwrap();
+               let next_update = update_iter.next().unwrap().clone();
+               // Should contain next_update when pending updates listed.
+               #[cfg(not(c_bindings))]
+               assert!(nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().get(funding_txo)
+                       .unwrap().contains(&next_update));
+               #[cfg(c_bindings)]
+               assert!(nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().iter()
+                       .find(|(txo, _)| txo == funding_txo).unwrap().1.contains(&next_update));
+               nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, next_update.clone()).unwrap();
+               // Should not contain the previously pending next_update when pending updates listed.
+               #[cfg(not(c_bindings))]
+               assert!(!nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().get(funding_txo)
+                       .unwrap().contains(&next_update));
+               #[cfg(c_bindings)]
+               assert!(!nodes[1].chain_monitor.chain_monitor.list_pending_monitor_updates().iter()
+                       .find(|(txo, _)| txo == funding_txo).unwrap().1.contains(&next_update));
                assert!(nodes[1].chain_monitor.release_pending_monitor_events().is_empty());
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
                nodes[1].chain_monitor.chain_monitor.channel_monitor_updated(*funding_txo, update_iter.next().unwrap().clone()).unwrap();
@@ -883,7 +933,7 @@ mod tests {
                // If the ChannelManager tries to update the channel, however, the ChainMonitor will pass
                // the update through to the ChannelMonitor which will refuse it (as the channel is closed).
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::Completed);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)),
+               unwrap_send_err!(nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)),
                        true, APIError::ChannelUnavailable { ref err },
                        assert!(err.contains("ChannelMonitor storage failure")));
                check_added_monitors!(nodes[0], 2); // After the failure we generate a close-channel monitor update
index 79d589c6521b0d0268d8a897d6950cdd97db0a96..c0d0affcce5d2fad616ab1ae48e49feaea2a76d7 100644 (file)
@@ -332,14 +332,15 @@ impl Readable for CounterpartyCommitmentParameters {
        }
 }
 
-/// An entry for an [`OnchainEvent`], stating the block height when the event was observed and the
-/// transaction causing it.
+/// An entry for an [`OnchainEvent`], stating the block height and hash when the event was
+/// observed, as well as the transaction causing it.
 ///
 /// Used to determine when the on-chain event can be considered safe from a chain reorganization.
 #[derive(PartialEq, Eq)]
 struct OnchainEventEntry {
        txid: Txid,
        height: u32,
+       block_hash: Option<BlockHash>, // Added as optional, will be filled in for any entry generated on 0.0.113 or after
        event: OnchainEvent,
        transaction: Option<Transaction>, // Added as optional, but always filled in, in LDK 0.0.110
 }
@@ -440,6 +441,7 @@ impl Writeable for OnchainEventEntry {
                        (0, self.txid, required),
                        (1, self.transaction, option),
                        (2, self.height, required),
+                       (3, self.block_hash, option),
                        (4, self.event, required),
                });
                Ok(())
@@ -450,16 +452,18 @@ impl MaybeReadable for OnchainEventEntry {
        fn read<R: io::Read>(reader: &mut R) -> Result<Option<Self>, DecodeError> {
                let mut txid = Txid::all_zeros();
                let mut transaction = None;
+               let mut block_hash = None;
                let mut height = 0;
                let mut event = None;
                read_tlv_fields!(reader, {
                        (0, txid, required),
                        (1, transaction, option),
                        (2, height, required),
+                       (3, block_hash, option),
                        (4, event, ignorable),
                });
                if let Some(ev) = event {
-                       Ok(Some(Self { txid, transaction, height, event: ev }))
+                       Ok(Some(Self { txid, transaction, height, block_hash, event: ev }))
                } else {
                        Ok(None)
                }
@@ -1305,9 +1309,11 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
        /// Gets the list of pending events which were generated by previous actions, clearing the list
        /// in the process.
        ///
-       /// This is called by ChainMonitor::get_and_clear_pending_events() and is equivalent to
-       /// EventsProvider::get_and_clear_pending_events() except that it requires &mut self as we do
-       /// no internal locking in ChannelMonitors.
+       /// This is called by the [`EventsProvider::process_pending_events`] implementation for
+       /// [`ChainMonitor`].
+       ///
+       /// [`EventsProvider::process_pending_events`]: crate::util::events::EventsProvider::process_pending_events
+       /// [`ChainMonitor`]: crate::chain::chainmonitor::ChainMonitor
        pub fn get_and_clear_pending_events(&self) -> Vec<Event> {
                self.inner.lock().unwrap().get_and_clear_pending_events()
        }
@@ -1482,11 +1488,11 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
        }
 
        /// Returns the set of txids that should be monitored for re-organization out of the chain.
-       pub fn get_relevant_txids(&self) -> Vec<Txid> {
+       pub fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let inner = self.inner.lock().unwrap();
-               let mut txids: Vec<Txid> = inner.onchain_events_awaiting_threshold_conf
+               let mut txids: Vec<(Txid, Option<BlockHash>)> = inner.onchain_events_awaiting_threshold_conf
                        .iter()
-                       .map(|entry| entry.txid)
+                       .map(|entry| (entry.txid, entry.block_hash))
                        .chain(inner.onchain_tx_handler.get_relevant_txids().into_iter())
                        .collect();
                txids.sort_unstable();
@@ -1939,7 +1945,7 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
 /// been revoked yet, the previous one, we we will never "forget" to resolve an HTLC.
 macro_rules! fail_unbroadcast_htlcs {
        ($self: expr, $commitment_tx_type: expr, $commitment_txid_confirmed: expr, $commitment_tx_confirmed: expr,
-        $commitment_tx_conf_height: expr, $confirmed_htlcs_list: expr, $logger: expr) => { {
+        $commitment_tx_conf_height: expr, $commitment_tx_conf_hash: expr, $confirmed_htlcs_list: expr, $logger: expr) => { {
                debug_assert_eq!($commitment_tx_confirmed.txid(), $commitment_txid_confirmed);
 
                macro_rules! check_htlc_fails {
@@ -1983,6 +1989,7 @@ macro_rules! fail_unbroadcast_htlcs {
                                                                txid: $commitment_txid_confirmed,
                                                                transaction: Some($commitment_tx_confirmed.clone()),
                                                                height: $commitment_tx_conf_height,
+                                                               block_hash: Some(*$commitment_tx_conf_hash),
                                                                event: OnchainEvent::HTLCUpdate {
                                                                        source: (**source).clone(),
                                                                        payment_hash: htlc.payment_hash.clone(),
@@ -2170,7 +2177,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                macro_rules! claim_htlcs {
                        ($commitment_number: expr, $txid: expr) => {
                                let (htlc_claim_reqs, _) = self.get_counterparty_output_claim_info($commitment_number, $txid, None);
-                               self.onchain_tx_handler.update_claims_view(&Vec::new(), htlc_claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
+                               self.onchain_tx_handler.update_claims_view_from_requests(htlc_claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
                        }
                }
                if let Some(txid) = self.current_counterparty_commitment_txid {
@@ -2196,10 +2203,10 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        // block. Even if not, its a reasonable metric for the bump criteria on the HTLC
                        // transactions.
                        let (claim_reqs, _) = self.get_broadcasted_holder_claims(&self.current_holder_commitment_tx, self.best_block.height());
-                       self.onchain_tx_handler.update_claims_view(&Vec::new(), claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
+                       self.onchain_tx_handler.update_claims_view_from_requests(claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
                        if let Some(ref tx) = self.prev_holder_signed_commitment_tx {
                                let (claim_reqs, _) = self.get_broadcasted_holder_claims(&tx, self.best_block.height());
-                               self.onchain_tx_handler.update_claims_view(&Vec::new(), claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
+                               self.onchain_tx_handler.update_claims_view_from_requests(claim_reqs, self.best_block.height(), self.best_block.height(), broadcaster, fee_estimator, logger);
                        }
                }
        }
@@ -2286,8 +2293,8 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                                PackageSolvingData::HolderFundingOutput(funding_output),
                                                                best_block_height, false, best_block_height,
                                                        );
-                                                       self.onchain_tx_handler.update_claims_view(
-                                                               &[], vec![commitment_package], best_block_height, best_block_height,
+                                                       self.onchain_tx_handler.update_claims_view_from_requests(
+                                                               vec![commitment_package], best_block_height, best_block_height,
                                                                broadcaster, &bounded_fee_estimator, logger,
                                                        );
                                                }
@@ -2401,7 +2408,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
        /// Returns packages to claim the revoked output(s), as well as additional outputs to watch and
        /// general information about the output that is to the counterparty in the commitment
        /// transaction.
-       fn check_spend_counterparty_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L)
+       fn check_spend_counterparty_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, block_hash: &BlockHash, logger: &L)
                -> (Vec<PackageTemplate>, TransactionOutputs, CommitmentTxCounterpartyOutputInfo)
        where L::Target: Logger {
                // Most secp and related errors trying to create keys means we have no hope of constructing
@@ -2472,13 +2479,13 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
 
                                if let Some(per_commitment_data) = per_commitment_option {
                                        fail_unbroadcast_htlcs!(self, "revoked_counterparty", commitment_txid, tx, height,
-                                               per_commitment_data.iter().map(|(htlc, htlc_source)|
+                                               block_hash, per_commitment_data.iter().map(|(htlc, htlc_source)|
                                                        (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref()))
                                                ), logger);
                                } else {
                                        debug_assert!(false, "We should have per-commitment option for any recognized old commitment txn");
                                        fail_unbroadcast_htlcs!(self, "revoked counterparty", commitment_txid, tx, height,
-                                               [].iter().map(|reference| *reference), logger);
+                                               block_hash, [].iter().map(|reference| *reference), logger);
                                }
                        }
                } else if let Some(per_commitment_data) = per_commitment_option {
@@ -2495,7 +2502,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        self.counterparty_commitment_txn_on_chain.insert(commitment_txid, commitment_number);
 
                        log_info!(logger, "Got broadcast of non-revoked counterparty commitment transaction {}", commitment_txid);
-                       fail_unbroadcast_htlcs!(self, "counterparty", commitment_txid, tx, height,
+                       fail_unbroadcast_htlcs!(self, "counterparty", commitment_txid, tx, height, block_hash,
                                per_commitment_data.iter().map(|(htlc, htlc_source)|
                                        (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref()))
                                ), logger);
@@ -2631,7 +2638,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                (claimable_outpoints, Some((htlc_txid, outputs)))
        }
 
-       // Returns (1) `PackageTemplate`s that can be given to the OnChainTxHandler, so that the handler can
+       // Returns (1) `PackageTemplate`s that can be given to the OnchainTxHandler, so that the handler can
        // broadcast transactions claiming holder HTLC commitment outputs and (2) a holder revokable
        // script so we can detect whether a holder transaction has been seen on-chain.
        fn get_broadcasted_holder_claims(&self, holder_tx: &HolderSignedTx, conf_height: u32) -> (Vec<PackageTemplate>, Option<(Script, PublicKey, PublicKey)>) {
@@ -2676,7 +2683,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
        /// revoked using data in holder_claimable_outpoints.
        /// Should not be used if check_spend_revoked_transaction succeeds.
        /// Returns None unless the transaction is definitely one of our commitment transactions.
-       fn check_spend_holder_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L) -> Option<(Vec<PackageTemplate>, TransactionOutputs)> where L::Target: Logger {
+       fn check_spend_holder_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, block_hash: &BlockHash, logger: &L) -> Option<(Vec<PackageTemplate>, TransactionOutputs)> where L::Target: Logger {
                let commitment_txid = tx.txid();
                let mut claim_requests = Vec::new();
                let mut watch_outputs = Vec::new();
@@ -2699,7 +2706,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        let mut to_watch = self.get_broadcasted_holder_watch_outputs(&self.current_holder_commitment_tx, tx);
                        append_onchain_update!(res, to_watch);
                        fail_unbroadcast_htlcs!(self, "latest holder", commitment_txid, tx, height,
-                               self.current_holder_commitment_tx.htlc_outputs.iter()
+                               block_hash, self.current_holder_commitment_tx.htlc_outputs.iter()
                                .map(|(htlc, _, htlc_source)| (htlc, htlc_source.as_ref())), logger);
                } else if let &Some(ref holder_tx) = &self.prev_holder_signed_commitment_tx {
                        if holder_tx.txid == commitment_txid {
@@ -2708,7 +2715,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                let res = self.get_broadcasted_holder_claims(holder_tx, height);
                                let mut to_watch = self.get_broadcasted_holder_watch_outputs(holder_tx, tx);
                                append_onchain_update!(res, to_watch);
-                               fail_unbroadcast_htlcs!(self, "previous holder", commitment_txid, tx, height,
+                               fail_unbroadcast_htlcs!(self, "previous holder", commitment_txid, tx, height, block_hash,
                                        holder_tx.htlc_outputs.iter().map(|(htlc, _, htlc_source)| (htlc, htlc_source.as_ref())),
                                        logger);
                        }
@@ -2816,7 +2823,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
 
                if height > self.best_block.height() {
                        self.best_block = BestBlock::new(block_hash, height);
-                       self.block_confirmed(height, vec![], vec![], vec![], &broadcaster, &fee_estimator, &logger)
+                       self.block_confirmed(height, block_hash, vec![], vec![], vec![], &broadcaster, &fee_estimator, &logger)
                } else if block_hash != self.best_block.block_hash() {
                        self.best_block = BestBlock::new(block_hash, height);
                        self.onchain_events_awaiting_threshold_conf.retain(|ref entry| entry.height <= height);
@@ -2868,14 +2875,14 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                        let mut commitment_tx_to_counterparty_output = None;
                                        if (tx.input[0].sequence.0 >> 8*3) as u8 == 0x80 && (tx.lock_time.0 >> 8*3) as u8 == 0x20 {
                                                let (mut new_outpoints, new_outputs, counterparty_output_idx_sats) =
-                                                       self.check_spend_counterparty_transaction(&tx, height, &logger);
+                                                       self.check_spend_counterparty_transaction(&tx, height, &block_hash, &logger);
                                                commitment_tx_to_counterparty_output = counterparty_output_idx_sats;
                                                if !new_outputs.1.is_empty() {
                                                        watch_outputs.push(new_outputs);
                                                }
                                                claimable_outpoints.append(&mut new_outpoints);
                                                if new_outpoints.is_empty() {
-                                                       if let Some((mut new_outpoints, new_outputs)) = self.check_spend_holder_transaction(&tx, height, &logger) {
+                                                       if let Some((mut new_outpoints, new_outputs)) = self.check_spend_holder_transaction(&tx, height, &block_hash, &logger) {
                                                                debug_assert!(commitment_tx_to_counterparty_output.is_none(),
                                                                        "A commitment transaction matched as both a counterparty and local commitment tx?");
                                                                if !new_outputs.1.is_empty() {
@@ -2891,6 +2898,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                txid,
                                                transaction: Some((*tx).clone()),
                                                height,
+                                               block_hash: Some(block_hash),
                                                event: OnchainEvent::FundingSpendConfirmation {
                                                        on_local_output_csv: balance_spendable_csv,
                                                        commitment_tx_to_counterparty_output,
@@ -2909,28 +2917,30 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        // While all commitment/HTLC-Success/HTLC-Timeout transactions have one input, HTLCs
                        // can also be resolved in a few other ways which can have more than one output. Thus,
                        // we call is_resolving_htlc_output here outside of the tx.input.len() == 1 check.
-                       self.is_resolving_htlc_output(&tx, height, &logger);
+                       self.is_resolving_htlc_output(&tx, height, &block_hash, &logger);
 
-                       self.is_paying_spendable_output(&tx, height, &logger);
+                       self.is_paying_spendable_output(&tx, height, &block_hash, &logger);
                }
 
                if height > self.best_block.height() {
                        self.best_block = BestBlock::new(block_hash, height);
                }
 
-               self.block_confirmed(height, txn_matched, watch_outputs, claimable_outpoints, &broadcaster, &fee_estimator, &logger)
+               self.block_confirmed(height, block_hash, txn_matched, watch_outputs, claimable_outpoints, &broadcaster, &fee_estimator, &logger)
        }
 
        /// Update state for new block(s)/transaction(s) confirmed. Note that the caller must update
        /// `self.best_block` before calling if a new best blockchain tip is available. More
        /// concretely, `self.best_block` must never be at a lower height than `conf_height`, avoiding
-       /// complexity especially in `OnchainTx::update_claims_view`.
+       /// complexity especially in
+       /// `OnchainTx::update_claims_view_from_requests`/`OnchainTx::update_claims_view_from_matched_txn`.
        ///
        /// `conf_height` should be set to the height at which any new transaction(s)/block(s) were
        /// confirmed at, even if it is not the current best height.
        fn block_confirmed<B: Deref, F: Deref, L: Deref>(
                &mut self,
                conf_height: u32,
+               conf_hash: BlockHash,
                txn_matched: Vec<&Transaction>,
                mut watch_outputs: Vec<TransactionOutputs>,
                mut claimable_outpoints: Vec<PackageTemplate>,
@@ -3046,7 +3056,8 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        }
                }
 
-               self.onchain_tx_handler.update_claims_view(&txn_matched, claimable_outpoints, conf_height, self.best_block.height(), broadcaster, fee_estimator, logger);
+               self.onchain_tx_handler.update_claims_view_from_requests(claimable_outpoints, conf_height, self.best_block.height(), broadcaster, fee_estimator, logger);
+               self.onchain_tx_handler.update_claims_view_from_matched_txn(&txn_matched, conf_height, conf_hash, self.best_block.height(), broadcaster, fee_estimator, logger);
 
                // Determine new outputs to watch by comparing against previously known outputs to watch,
                // updating the latter in the process.
@@ -3235,7 +3246,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
 
        /// Check if any transaction broadcasted is resolving HTLC output by a success or timeout on a holder
        /// or counterparty commitment tx, if so send back the source, preimage if found and payment_hash of resolved HTLC
-       fn is_resolving_htlc_output<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L) where L::Target: Logger {
+       fn is_resolving_htlc_output<L: Deref>(&mut self, tx: &Transaction, height: u32, block_hash: &BlockHash, logger: &L) where L::Target: Logger {
                'outer_loop: for input in &tx.input {
                        let mut payment_data = None;
                        let htlc_claim = HTLCClaim::from_witness(&input.witness);
@@ -3320,7 +3331,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                                log_claim!($tx_info, $holder_tx, htlc_output, false);
                                                                let outbound_htlc = $holder_tx == htlc_output.offered;
                                                                self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
-                                                                       txid: tx.txid(), height, transaction: Some(tx.clone()),
+                                                                       txid: tx.txid(), height, block_hash: Some(*block_hash), transaction: Some(tx.clone()),
                                                                        event: OnchainEvent::HTLCSpendConfirmation {
                                                                                commitment_tx_output_idx: input.previous_output.vout,
                                                                                preimage: if accepted_preimage_claim || offered_preimage_claim {
@@ -3364,6 +3375,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
                                                        txid: tx.txid(),
                                                        height,
+                                                       block_hash: Some(*block_hash),
                                                        transaction: Some(tx.clone()),
                                                        event: OnchainEvent::HTLCSpendConfirmation {
                                                                commitment_tx_output_idx: input.previous_output.vout,
@@ -3387,6 +3399,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                        txid: tx.txid(),
                                                        transaction: Some(tx.clone()),
                                                        height,
+                                                       block_hash: Some(*block_hash),
                                                        event: OnchainEvent::HTLCSpendConfirmation {
                                                                commitment_tx_output_idx: input.previous_output.vout,
                                                                preimage: Some(payment_preimage),
@@ -3414,6 +3427,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                txid: tx.txid(),
                                                transaction: Some(tx.clone()),
                                                height,
+                                               block_hash: Some(*block_hash),
                                                event: OnchainEvent::HTLCUpdate {
                                                        source, payment_hash,
                                                        htlc_value_satoshis: Some(amount_msat / 1000),
@@ -3428,7 +3442,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
        }
 
        /// Check if any transaction broadcasted is paying fund back to some address we can assume to own
-       fn is_paying_spendable_output<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L) where L::Target: Logger {
+       fn is_paying_spendable_output<L: Deref>(&mut self, tx: &Transaction, height: u32, block_hash: &BlockHash, logger: &L) where L::Target: Logger {
                let mut spendable_output = None;
                for (i, outp) in tx.output.iter().enumerate() { // There is max one spendable output for any channel tx, including ones generated by us
                        if i > ::core::u16::MAX as usize {
@@ -3488,6 +3502,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                txid: tx.txid(),
                                transaction: Some(tx.clone()),
                                height,
+                               block_hash: Some(*block_hash),
                                event: OnchainEvent::MaturingOutput { descriptor: spendable_output.clone() },
                        };
                        log_info!(logger, "Received spendable output {}, spendable at height {}", log_spendable!(spendable_output), entry.confirmation_threshold());
@@ -3529,15 +3544,15 @@ where
                self.0.best_block_updated(header, height, &*self.1, &*self.2, &*self.3);
        }
 
-       fn get_relevant_txids(&self) -> Vec<Txid> {
+       fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                self.0.get_relevant_txids()
        }
 }
 
 const MAX_ALLOC_SIZE: usize = 64*1024;
 
-impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
-               for (BlockHash, ChannelMonitor<Signer>) {
+impl<'a, K: KeysInterface> ReadableArgs<&'a K>
+               for (BlockHash, ChannelMonitor<K::Signer>) {
        fn read<R: io::Read>(reader: &mut R, keys_manager: &'a K) -> Result<Self, DecodeError> {
                macro_rules! unwrap_obj {
                        ($key: expr) => {
@@ -3721,7 +3736,7 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                                return Err(DecodeError::InvalidValue);
                        }
                }
-               let onchain_tx_handler: OnchainTxHandler<Signer> = ReadableArgs::read(reader, keys_manager)?;
+               let onchain_tx_handler: OnchainTxHandler<K::Signer> = ReadableArgs::read(reader, keys_manager)?;
 
                let lockdown_from_offchain = Readable::read(reader)?;
                let holder_tx_signed = Readable::read(reader)?;
@@ -3846,7 +3861,7 @@ mod tests {
        use crate::ln::{PaymentPreimage, PaymentHash};
        use crate::ln::chan_utils;
        use crate::ln::chan_utils::{HTLCOutputInCommitment, ChannelPublicKeys, ChannelTransactionParameters, HolderCommitmentTransaction, CounterpartyChannelTransactionParameters};
-       use crate::ln::channelmanager::{self, PaymentSendFailure};
+       use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId};
        use crate::ln::functional_test_utils::*;
        use crate::ln::script::ShutdownScript;
        use crate::util::errors::APIError;
@@ -3911,7 +3926,7 @@ mod tests {
                // If the ChannelManager tries to update the channel, however, the ChainMonitor will pass
                // the update through to the ChannelMonitor which will refuse it (as the channel is closed).
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 100_000);
-               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)),
+               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
                        true, APIError::ChannelUnavailable { ref err },
                        assert!(err.contains("ChannelMonitor storage failure")));
                check_added_monitors!(nodes[1], 2); // After the failure we generate a close-channel monitor update
index e3108fd54daa9c5a773cf0acfa85611c3c5fd787..b12eee017aa93b81a080bf342a267c76aa309f5e 100644 (file)
@@ -352,7 +352,7 @@ pub trait BaseSign {
        /// Computes the signature for a commitment transaction's anchor output used as an
        /// input within `anchor_tx`, which spends the commitment transaction, at index `input`.
        fn sign_holder_anchor_input(
-               &self, anchor_tx: &mut Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
+               &self, anchor_tx: &Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
        ) -> Result<Signature, ()>;
 
        /// Signs a channel announcement message with our funding key and our node secret key (aka
@@ -790,7 +790,7 @@ impl BaseSign for InMemorySigner {
        }
 
        fn sign_holder_anchor_input(
-               &self, anchor_tx: &mut Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
+               &self, anchor_tx: &Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
        ) -> Result<Signature, ()> {
                let witness_script = chan_utils::get_anchor_redeemscript(&self.holder_channel_pubkeys.funding_pubkey);
                let sighash = sighash::SighashCache::new(&*anchor_tx).segwit_signature_hash(
index 8ac6c39360b9624043115756ce85461144e3fb8d..33d4826fef210b849b61818fd9ba241aad2a9971 100644 (file)
@@ -171,7 +171,8 @@ pub trait Confirm {
        /// if they become available at the same time.
        fn best_block_updated(&self, header: &BlockHeader, height: u32);
 
-       /// Returns transactions that should be monitored for reorganization out of the chain.
+       /// Returns transactions that should be monitored for reorganization out of the chain along
+       /// with the hash of the block as part of which had been previously confirmed.
        ///
        /// Will include any transactions passed to [`transactions_confirmed`] that have insufficient
        /// confirmations to be safe from a chain reorganization. Will not include any transactions
@@ -180,11 +181,13 @@ pub trait Confirm {
        /// May be called to determine the subset of transactions that must still be monitored for
        /// reorganization. Will be idempotent between calls but may change as a result of calls to the
        /// other interface methods. Thus, this is useful to determine which transactions may need to be
-       /// given to [`transaction_unconfirmed`].
+       /// given to [`transaction_unconfirmed`]. If any of the returned transactions are confirmed in
+       /// a block other than the one with the given hash, they need to be unconfirmed and reconfirmed
+       /// via [`transaction_unconfirmed`] and [`transactions_confirmed`], respectively.
        ///
        /// [`transactions_confirmed`]: Self::transactions_confirmed
        /// [`transaction_unconfirmed`]: Self::transaction_unconfirmed
-       fn get_relevant_txids(&self) -> Vec<Txid>;
+       fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)>;
 }
 
 /// An enum representing the status of a channel monitor update persistence.
index 18679f0aacd5cd34fead7952fc91be997ad4e57b..2ce2ed41ba1fcf0e00325aec5005e8757d10c76b 100644 (file)
@@ -16,7 +16,7 @@ use bitcoin::blockdata::transaction::Transaction;
 use bitcoin::blockdata::transaction::OutPoint as BitcoinOutPoint;
 use bitcoin::blockdata::script::Script;
 
-use bitcoin::hash_types::Txid;
+use bitcoin::hash_types::{Txid, BlockHash};
 
 use bitcoin::secp256k1::{Secp256k1, ecdsa::Signature};
 use bitcoin::secp256k1;
@@ -58,6 +58,7 @@ const MAX_ALLOC_SIZE: usize = 64*1024;
 struct OnchainEventEntry {
        txid: Txid,
        height: u32,
+       block_hash: Option<BlockHash>, // Added as optional, will be filled in for any entry generated on 0.0.113 or after
        event: OnchainEvent,
 }
 
@@ -92,6 +93,7 @@ impl Writeable for OnchainEventEntry {
        fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
                write_tlv_fields!(writer, {
                        (0, self.txid, required),
+                       (1, self.block_hash, option),
                        (2, self.height, required),
                        (4, self.event, required),
                });
@@ -103,14 +105,16 @@ impl MaybeReadable for OnchainEventEntry {
        fn read<R: io::Read>(reader: &mut R) -> Result<Option<Self>, DecodeError> {
                let mut txid = Txid::all_zeros();
                let mut height = 0;
+               let mut block_hash = None;
                let mut event = None;
                read_tlv_fields!(reader, {
                        (0, txid, required),
+                       (1, block_hash, option),
                        (2, height, required),
                        (4, event, ignorable),
                });
                if let Some(ev) = event {
-                       Ok(Some(Self { txid, height, event: ev }))
+                       Ok(Some(Self { txid, height, block_hash, event: ev }))
                } else {
                        Ok(None)
                }
@@ -430,7 +434,43 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                where F::Target: FeeEstimator,
                                        L::Target: Logger,
        {
-               if cached_request.outpoints().len() == 0 { return None } // But don't prune pending claiming request yet, we may have to resurrect HTLCs
+               let request_outpoints = cached_request.outpoints();
+               if request_outpoints.is_empty() {
+                       // Don't prune pending claiming request yet, we may have to resurrect HTLCs. Untractable
+                       // packages cannot be aggregated and will never be split, so we cannot end up with an
+                       // empty claim.
+                       debug_assert!(cached_request.is_malleable());
+                       return None;
+               }
+               // If we've seen transaction inclusion in the chain for all outpoints in our request, we
+               // don't need to continue generating more claims. We'll keep tracking the request to fully
+               // remove it once it reaches the confirmation threshold, or to generate a new claim if the
+               // transaction is reorged out.
+               let mut all_inputs_have_confirmed_spend = true;
+               for outpoint in &request_outpoints {
+                       if let Some(first_claim_txid_height) = self.claimable_outpoints.get(outpoint) {
+                               // We check for outpoint spends within claims individually rather than as a set
+                               // since requests can have outpoints split off.
+                               if !self.onchain_events_awaiting_threshold_conf.iter()
+                                       .any(|event_entry| if let OnchainEvent::Claim { claim_request } = event_entry.event {
+                                               first_claim_txid_height.0 == claim_request
+                                       } else {
+                                               // The onchain event is not a claim, keep seeking until we find one.
+                                               false
+                                       })
+                               {
+                                       // Either we had no `OnchainEvent::Claim`, or we did but none matched the
+                                       // outpoint's registered spend.
+                                       all_inputs_have_confirmed_spend = false;
+                               }
+                       } else {
+                               // The request's outpoint spend does not exist yet.
+                               all_inputs_have_confirmed_spend = false;
+                       }
+               }
+               if all_inputs_have_confirmed_spend {
+                       return None;
+               }
 
                // Compute new height timer to decide when we need to regenerate a new bumped version of the claim tx (if we
                // didn't receive confirmation of it before, or not enough reorg-safe depth on top of it).
@@ -507,17 +547,22 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
 
        /// Upon channelmonitor.block_connected(..) or upon provision of a preimage on the forward link
        /// for this channel, provide new relevant on-chain transactions and/or new claim requests.
-       /// Formerly this was named `block_connected`, but it is now also used for claiming an HTLC output
-       /// if we receive a preimage after force-close.
-       /// `conf_height` represents the height at which the transactions in `txn_matched` were
-       /// confirmed. This does not need to equal the current blockchain tip height, which should be
-       /// provided via `cur_height`, however it must never be higher than `cur_height`.
-       pub(crate) fn update_claims_view<B: Deref, F: Deref, L: Deref>(&mut self, txn_matched: &[&Transaction], requests: Vec<PackageTemplate>, conf_height: u32, cur_height: u32, broadcaster: &B, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L)
-               where B::Target: BroadcasterInterface,
-                     F::Target: FeeEstimator,
-                                       L::Target: Logger,
+       /// Together with `update_claims_view_from_matched_txn` this used to be named
+       /// `block_connected`, but it is now also used for claiming an HTLC output if we receive a
+       /// preimage after force-close.
+       ///
+       /// `conf_height` represents the height at which the request was generated. This
+       /// does not need to equal the current blockchain tip height, which should be provided via
+       /// `cur_height`, however it must never be higher than `cur_height`.
+       pub(crate) fn update_claims_view_from_requests<B: Deref, F: Deref, L: Deref>(
+               &mut self, requests: Vec<PackageTemplate>, conf_height: u32, cur_height: u32,
+               broadcaster: &B, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L
+       ) where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
        {
-               log_debug!(logger, "Updating claims view at height {} with {} matched transactions in block {} and {} claim requests", cur_height, txn_matched.len(), conf_height, requests.len());
+               log_debug!(logger, "Updating claims view at height {} with {} claim requests", cur_height, requests.len());
                let mut preprocessed_requests = Vec::with_capacity(requests.len());
                let mut aggregated_request = None;
 
@@ -597,7 +642,25 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                self.pending_claim_requests.insert(txid, req);
                        }
                }
+       }
 
+       /// Upon channelmonitor.block_connected(..) or upon provision of a preimage on the forward link
+       /// for this channel, provide new relevant on-chain transactions and/or new claim requests.
+       /// Together with `update_claims_view_from_requests` this used to be named `block_connected`,
+       /// but it is now also used for claiming an HTLC output if we receive a preimage after force-close.
+       ///
+       /// `conf_height` represents the height at which the transactions in `txn_matched` were
+       /// confirmed. This does not need to equal the current blockchain tip height, which should be
+       /// provided via `cur_height`, however it must never be higher than `cur_height`.
+       pub(crate) fn update_claims_view_from_matched_txn<B: Deref, F: Deref, L: Deref>(
+               &mut self, txn_matched: &[&Transaction], conf_height: u32, conf_hash: BlockHash,
+               cur_height: u32, broadcaster: &B, fee_estimator: &LowerBoundedFeeEstimator<F>, logger: &L
+       ) where
+               B::Target: BroadcasterInterface,
+               F::Target: FeeEstimator,
+               L::Target: Logger,
+       {
+               log_debug!(logger, "Updating claims view at height {} with {} matched transactions in block {}", cur_height, txn_matched.len(), conf_height);
                let mut bump_candidates = HashMap::new();
                for tx in txn_matched {
                        // Scan all input to verify is one of the outpoint spent is of interest for us
@@ -625,6 +688,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                                                let entry = OnchainEventEntry {
                                                                        txid: tx.txid(),
                                                                        height: conf_height,
+                                                                       block_hash: Some(conf_hash),
                                                                        event: OnchainEvent::Claim { claim_request: first_claim_txid_height.0.clone() }
                                                                };
                                                                if !self.onchain_events_awaiting_threshold_conf.contains(&entry) {
@@ -665,6 +729,7 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                                let entry = OnchainEventEntry {
                                        txid: tx.txid(),
                                        height: conf_height,
+                                       block_hash: Some(conf_hash),
                                        event: OnchainEvent::ContentiousOutpoint { package },
                                };
                                if !self.onchain_events_awaiting_threshold_conf.contains(&entry) {
@@ -824,12 +889,12 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                self.claimable_outpoints.get(outpoint).is_some()
        }
 
-       pub(crate) fn get_relevant_txids(&self) -> Vec<Txid> {
-               let mut txids: Vec<Txid> = self.onchain_events_awaiting_threshold_conf
+       pub(crate) fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
+               let mut txids: Vec<(Txid, Option<BlockHash>)> = self.onchain_events_awaiting_threshold_conf
                        .iter()
-                       .map(|entry| entry.txid)
+                       .map(|entry| (entry.txid, entry.block_hash))
                        .collect();
-               txids.sort_unstable();
+               txids.sort_unstable_by_key(|(txid, _)| *txid);
                txids.dedup();
                txids
        }
index 25eba1d74f8f87ddb60fd80d610b187f853f81ce..1f3ab47b1ae3f7d2ca8802e46c267d5428e3822c 100644 (file)
@@ -78,6 +78,8 @@ extern crate core;
 pub mod util;
 pub mod chain;
 pub mod ln;
+#[allow(unused)]
+mod offers;
 pub mod routing;
 pub mod onion_message;
 
index f16875252428499fe2122e90e28c5cf57c866e30..368b2e5ff82070baf1c45f3dc1e9fa92bdf2a987 100644 (file)
@@ -19,7 +19,7 @@ use bitcoin::network::constants::Network;
 use crate::chain::channelmonitor::{ANTI_REORG_DELAY, ChannelMonitor};
 use crate::chain::transaction::OutPoint;
 use crate::chain::{ChannelMonitorUpdateStatus, Listen, Watch};
-use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure};
+use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, PaymentId};
 use crate::ln::channel::AnnouncementSigsState;
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler};
@@ -51,7 +51,7 @@ fn test_simple_monitor_permanent_update_fail() {
 
        let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(&nodes[0], nodes[1], 1000000);
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::PermanentFailure);
-       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), true, APIError::ChannelUnavailable {..}, {});
        check_added_monitors!(nodes[0], 2);
 
        let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
@@ -170,7 +170,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
 
        {
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -221,7 +221,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(&nodes[0], nodes[1], 1000000);
        {
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -285,7 +285,7 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
                chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateInProgress, {});
+               unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)), false, APIError::MonitorUpdateInProgress, {});
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -624,7 +624,7 @@ fn test_monitor_update_fail_cs() {
 
        let (route, our_payment_hash, payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -716,7 +716,7 @@ fn test_monitor_update_fail_no_rebroadcast() {
 
        let (route, our_payment_hash, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(payment_secret_1), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -764,14 +764,14 @@ fn test_monitor_update_raa_while_paused() {
        send_payment(&nodes[0], &[&nodes[1]], 5000000);
        let (route, our_payment_hash_1, payment_preimage_1, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, our_payment_hash_1, &Some(our_payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
        let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 
        let (route, our_payment_hash_2, payment_preimage_2, our_payment_secret_2) = get_route_and_payment_hash!(nodes[1], nodes[0], 1000000);
        {
-               nodes[1].node.send_payment(&route, our_payment_hash_2, &Some(our_payment_secret_2)).unwrap();
+               nodes[1].node.send_payment(&route, our_payment_hash_2, &Some(our_payment_secret_2), PaymentId(our_payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
        }
        let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
@@ -860,7 +860,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        // holding cell.
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -885,7 +885,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        // being paused waiting a monitor update.
        let (route, payment_hash_3, _, payment_secret_3) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -904,7 +904,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
                // Try to route another payment backwards from 2 to make sure 1 holds off on responding
                let (route, payment_hash_4, payment_preimage_4, payment_secret_4) = get_route_and_payment_hash!(nodes[2], nodes[0], 1000000);
-               nodes[2].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4)).unwrap();
+               nodes[2].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
 
                send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
@@ -1198,9 +1198,9 @@ fn raa_no_response_awaiting_raa_state() {
        // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
        // generation during RAA while in monitor-update-failed state.
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
        }
 
@@ -1250,7 +1250,7 @@ fn raa_no_response_awaiting_raa_state() {
        // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
        // commitment transaction states) whereas here we can explicitly check for it.
        {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        }
@@ -1340,7 +1340,7 @@ fn claim_while_disconnected_monitor_update_fail() {
        // the monitor still failed
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1430,7 +1430,7 @@ fn monitor_failed_no_reestablish_response() {
        // on receipt).
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1503,7 +1503,7 @@ fn first_message_on_recv_ordering() {
        // can deliver it and fail the monitor update.
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1526,7 +1526,7 @@ fn first_message_on_recv_ordering() {
        // Route the second payment, generating an update_add_htlc/commitment_signed
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1610,7 +1610,7 @@ fn test_monitor_update_fail_claim() {
 
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[2], nodes[0], 1_000_000);
        {
-               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
 
@@ -1627,7 +1627,7 @@ fn test_monitor_update_fail_claim() {
        commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
 
        let (_, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[0]);
-       nodes[2].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
+       nodes[2].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
        check_added_monitors!(nodes[2], 1);
 
        let mut events = nodes[2].node.get_and_clear_pending_msg_events();
@@ -1717,7 +1717,7 @@ fn test_monitor_update_on_pending_forwards() {
 
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[2], nodes[0], 1000000);
        {
-               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
 
@@ -1777,7 +1777,7 @@ fn monitor_update_claim_fail_no_response() {
        // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1916,6 +1916,13 @@ fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf:
                node.gossip_sync.handle_channel_update(&bs_update).unwrap();
        }
 
+       if !restore_b_before_lock {
+               expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
+       } else {
+               expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
+       }
+
+
        send_payment(&nodes[0], &[&nodes[1]], 8000000);
        close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
        check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
@@ -1964,7 +1971,7 @@ fn test_path_paused_mpp() {
        // Now check that we get the right return value, indicating that the first path succeeded but
        // the second got a MonitorUpdateInProgress err. This implies
        // PaymentSendFailure::PartialFailure as some paths succeeded, preventing retry.
-       if let Err(PaymentSendFailure::PartialFailure { results, ..}) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
+       if let Err(PaymentSendFailure::PartialFailure { results, ..}) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) {
                assert_eq!(results.len(), 2);
                if let Ok(()) = results[0] {} else { panic!(); }
                if let Err(APIError::MonitorUpdateInProgress) = results[1] {} else { panic!(); }
@@ -2009,7 +2016,7 @@ fn test_pending_update_fee_ack_on_reconnect() {
        send_payment(&nodes[0], &[&nodes[1]], 100_000_00);
 
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[1], nodes[0], 1_000_000);
-       nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
        let bs_initial_send_msgs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
        // bs_initial_send_msgs are not delivered until they are re-generated after reconnect
@@ -2231,7 +2238,7 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let chan_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 15_000_000, 7_000_000_000, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
@@ -2260,12 +2267,12 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
        // (c) will not be freed from the holding cell.
        let (payment_preimage_0, payment_hash_0, _) = route_payment(&nodes[1], &[&nodes[0]], 100_000);
 
-       nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let send = SendEvent::from_node(&nodes[0]);
        assert_eq!(send.msgs.len(), 1);
 
-       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
 
        chanmon_cfgs[0].persister.set_update_ret(ChannelMonitorUpdateStatus::InProgress);
@@ -2305,7 +2312,7 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
                        nodes_0_deserialized = {
                                let mut channel_monitors = HashMap::new();
                                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-                               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+                               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                                        default_config: config,
                                        keys_manager,
                                        fee_estimator: node_cfgs[0].fee_estimator,
@@ -2457,7 +2464,7 @@ fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_f
                // In order to get the HTLC claim into the holding cell at nodes[1], we need nodes[1] to be
                // awaiting a remote revoke_and_ack from nodes[0].
                let (route, second_payment_hash, _, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
@@ -2756,7 +2763,7 @@ fn do_test_outbound_reload_without_init_mon(use_0conf: bool) {
 
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 
        let mut chan_config = test_default_channel_config();
        chan_config.manually_accept_inbound_channels = true;
@@ -2834,7 +2841,7 @@ fn do_test_outbound_reload_without_init_mon(use_0conf: bool) {
        let mut nodes_0_read = &nodes_0_serialized[..];
        let config = UserConfig::default();
        nodes_0_deserialized = {
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: config,
                        keys_manager,
                        fee_estimator: node_cfgs[0].fee_estimator,
@@ -2866,7 +2873,7 @@ fn do_test_inbound_reload_without_init_mon(use_0conf: bool, lock_commitment: boo
 
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_1_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 
        let mut chan_config = test_default_channel_config();
        chan_config.manually_accept_inbound_channels = true;
@@ -2940,7 +2947,7 @@ fn do_test_inbound_reload_without_init_mon(use_0conf: bool, lock_commitment: boo
        let mut nodes_1_read = &nodes_1_serialized[..];
        let config = UserConfig::default();
        nodes_1_deserialized = {
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
                        default_config: config,
                        keys_manager,
                        fee_estimator: node_cfgs[1].fee_estimator,
index 7af90c4f843e00ee1bf4aa30aa0eb0796ebbf15b..0e304f3074524f56a537681279c558e685a9354d 100644 (file)
@@ -244,9 +244,9 @@ enum HTLCUpdateAwaitingACK {
 /// There are a few "states" and then a number of flags which can be applied:
 /// We first move through init with OurInitSent -> TheirInitSent -> FundingCreated -> FundingSent.
 /// TheirChannelReady and OurChannelReady then get set on FundingSent, and when both are set we
-/// move on to ChannelFunded.
-/// Note that PeerDisconnected can be set on both ChannelFunded and FundingSent.
-/// ChannelFunded can then get all remaining flags set on it, until we finish shutdown, then we
+/// move on to ChannelReady.
+/// Note that PeerDisconnected can be set on both ChannelReady and FundingSent.
+/// ChannelReady can then get all remaining flags set on it, until we finish shutdown, then we
 /// move on to ShutdownComplete, at which point most calls into this channel are disallowed.
 enum ChannelState {
        /// Implies we have (or are prepared to) send our open_channel/accept_channel message
@@ -262,17 +262,17 @@ enum ChannelState {
        /// and our counterparty consider the funding transaction confirmed.
        FundingSent = 8,
        /// Flag which can be set on FundingSent to indicate they sent us a channel_ready message.
-       /// Once both TheirChannelReady and OurChannelReady are set, state moves on to ChannelFunded.
+       /// Once both TheirChannelReady and OurChannelReady are set, state moves on to ChannelReady.
        TheirChannelReady = 1 << 4,
        /// Flag which can be set on FundingSent to indicate we sent them a channel_ready message.
-       /// Once both TheirChannelReady and OurChannelReady are set, state moves on to ChannelFunded.
+       /// Once both TheirChannelReady and OurChannelReady are set, state moves on to ChannelReady.
        OurChannelReady = 1 << 5,
-       ChannelFunded = 64,
-       /// Flag which is set on ChannelFunded and FundingSent indicating remote side is considered
+       ChannelReady = 64,
+       /// Flag which is set on ChannelReady and FundingSent indicating remote side is considered
        /// "disconnected" and no updates are allowed until after we've done a channel_reestablish
        /// dance.
        PeerDisconnected = 1 << 7,
-       /// Flag which is set on ChannelFunded, FundingCreated, and FundingSent indicating the user has
+       /// Flag which is set on ChannelReady, FundingCreated, and FundingSent indicating the user has
        /// told us a ChannelMonitor update is pending async persistence somewhere and we should pause
        /// sending any outbound messages until they've managed to finish.
        MonitorUpdateInProgress = 1 << 8,
@@ -281,13 +281,13 @@ enum ChannelState {
        /// messages as then we will be unable to determine which HTLCs they included in their
        /// revoke_and_ack implicit ACK, so instead we have to hold them away temporarily to be sent
        /// later.
-       /// Flag is set on ChannelFunded.
+       /// Flag is set on ChannelReady.
        AwaitingRemoteRevoke = 1 << 9,
-       /// Flag which is set on ChannelFunded or FundingSent after receiving a shutdown message from
+       /// Flag which is set on ChannelReady or FundingSent after receiving a shutdown message from
        /// the remote end. If set, they may not add any new HTLCs to the channel, and we are expected
        /// to respond with our own shutdown message when possible.
        RemoteShutdownSent = 1 << 10,
-       /// Flag which is set on ChannelFunded or FundingSent after sending a shutdown message. At this
+       /// Flag which is set on ChannelReady or FundingSent after sending a shutdown message. At this
        /// point, we may not add any new HTLCs to the channel.
        LocalShutdownSent = 1 << 11,
        /// We've successfully negotiated a closing_signed dance. At this point ChannelManager is about
@@ -736,6 +736,9 @@ pub(super) struct Channel<Signer: Sign> {
        // don't currently support node id aliases and eventually privacy should be provided with
        // blinded paths instead of simple scid+node_id aliases.
        outbound_scid_alias: u64,
+
+       // We track whether we already emitted a `ChannelReady` event.
+       channel_ready_event_emitted: bool,
 }
 
 #[cfg(any(test, fuzzing))]
@@ -1063,6 +1066,8 @@ impl<Signer: Sign> Channel<Signer> {
                        latest_inbound_scid_alias: None,
                        outbound_scid_alias,
 
+                       channel_ready_event_emitted: false,
+
                        #[cfg(any(test, fuzzing))]
                        historical_inbound_htlc_fulfills: HashSet::new(),
 
@@ -1397,6 +1402,8 @@ impl<Signer: Sign> Channel<Signer> {
                        latest_inbound_scid_alias: None,
                        outbound_scid_alias,
 
+                       channel_ready_event_emitted: false,
+
                        #[cfg(any(test, fuzzing))]
                        historical_inbound_htlc_fulfills: HashSet::new(),
 
@@ -1786,11 +1793,11 @@ impl<Signer: Sign> Channel<Signer> {
        }
 
        fn get_update_fulfill_htlc<L: Deref>(&mut self, htlc_id_arg: u64, payment_preimage_arg: PaymentPreimage, logger: &L) -> UpdateFulfillFetch where L::Target: Logger {
-               // Either ChannelFunded got set (which means it won't be unset) or there is no way any
+               // Either ChannelReady got set (which means it won't be unset) or there is no way any
                // caller thought we could have something claimed (cause we wouldn't have accepted in an
                // incoming HTLC anyway). If we got to ShutdownComplete, callers aren't allowed to call us,
                // either.
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        panic!("Was asked to fulfill an HTLC when channel was not in an operational state");
                }
                assert_eq!(self.channel_state & ChannelState::ShutdownComplete as u32, 0);
@@ -1933,7 +1940,7 @@ impl<Signer: Sign> Channel<Signer> {
        /// If we do fail twice, we debug_assert!(false) and return Ok(None). Thus, will always return
        /// Ok(_) if debug assertions are turned on or preconditions are met.
        pub fn get_update_fail_htlc<L: Deref>(&mut self, htlc_id_arg: u64, err_packet: msgs::OnionErrorPacket, logger: &L) -> Result<Option<msgs::UpdateFailHTLC>, ChannelError> where L::Target: Logger {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        panic!("Was asked to fail an HTLC when channel was not in an operational state");
                }
                assert_eq!(self.channel_state & ChannelState::ShutdownComplete as u32, 0);
@@ -2349,9 +2356,9 @@ impl<Signer: Sign> Channel<Signer> {
                if non_shutdown_state == ChannelState::FundingSent as u32 {
                        self.channel_state |= ChannelState::TheirChannelReady as u32;
                } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::OurChannelReady as u32) {
-                       self.channel_state = ChannelState::ChannelFunded as u32 | (self.channel_state & MULTI_STATE_FLAGS);
+                       self.channel_state = ChannelState::ChannelReady as u32 | (self.channel_state & MULTI_STATE_FLAGS);
                        self.update_time_counter += 1;
-               } else if self.channel_state & (ChannelState::ChannelFunded as u32) != 0 ||
+               } else if self.channel_state & (ChannelState::ChannelReady as u32) != 0 ||
                        // If we reconnected before sending our `channel_ready` they may still resend theirs:
                        (self.channel_state & (ChannelState::FundingSent as u32 | ChannelState::TheirChannelReady as u32) ==
                                              (ChannelState::FundingSent as u32 | ChannelState::TheirChannelReady as u32))
@@ -2712,12 +2719,12 @@ impl<Signer: Sign> Channel<Signer> {
        pub fn update_add_htlc<F, L: Deref>(&mut self, msg: &msgs::UpdateAddHTLC, mut pending_forward_status: PendingHTLCStatus, create_pending_htlc_status: F, logger: &L) -> Result<(), ChannelError>
        where F: for<'a> Fn(&'a Self, PendingHTLCStatus, u16) -> PendingHTLCStatus, L::Target: Logger {
                // We can't accept HTLCs sent after we've sent a shutdown.
-               let local_sent_shutdown = (self.channel_state & (ChannelState::ChannelFunded as u32 | ChannelState::LocalShutdownSent as u32)) != (ChannelState::ChannelFunded as u32);
+               let local_sent_shutdown = (self.channel_state & (ChannelState::ChannelReady as u32 | ChannelState::LocalShutdownSent as u32)) != (ChannelState::ChannelReady as u32);
                if local_sent_shutdown {
                        pending_forward_status = create_pending_htlc_status(self, pending_forward_status, 0x4000|8);
                }
                // If the remote has sent a shutdown prior to adding this HTLC, then they are in violation of the spec.
-               let remote_sent_shutdown = (self.channel_state & (ChannelState::ChannelFunded as u32 | ChannelState::RemoteShutdownSent as u32)) != (ChannelState::ChannelFunded as u32);
+               let remote_sent_shutdown = (self.channel_state & (ChannelState::ChannelReady as u32 | ChannelState::RemoteShutdownSent as u32)) != (ChannelState::ChannelReady as u32);
                if remote_sent_shutdown {
                        return Err(ChannelError::Close("Got add HTLC message when channel was not in an operational state".to_owned()));
                }
@@ -2894,7 +2901,7 @@ impl<Signer: Sign> Channel<Signer> {
        }
 
        pub fn update_fulfill_htlc(&mut self, msg: &msgs::UpdateFulfillHTLC) -> Result<(HTLCSource, u64), ChannelError> {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Close("Got fulfill HTLC message when channel was not in an operational state".to_owned()));
                }
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
@@ -2905,7 +2912,7 @@ impl<Signer: Sign> Channel<Signer> {
        }
 
        pub fn update_fail_htlc(&mut self, msg: &msgs::UpdateFailHTLC, fail_reason: HTLCFailReason) -> Result<(), ChannelError> {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Close("Got fail HTLC message when channel was not in an operational state".to_owned()));
                }
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
@@ -2917,7 +2924,7 @@ impl<Signer: Sign> Channel<Signer> {
        }
 
        pub fn update_fail_malformed_htlc(&mut self, msg: &msgs::UpdateFailMalformedHTLC, fail_reason: HTLCFailReason) -> Result<(), ChannelError> {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Close("Got fail malformed HTLC message when channel was not in an operational state".to_owned()));
                }
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
@@ -2931,7 +2938,7 @@ impl<Signer: Sign> Channel<Signer> {
        pub fn commitment_signed<L: Deref>(&mut self, msg: &msgs::CommitmentSigned, logger: &L) -> Result<(msgs::RevokeAndACK, Option<msgs::CommitmentSigned>, ChannelMonitorUpdate), (Option<ChannelMonitorUpdate>, ChannelError)>
                where L::Target: Logger
        {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        return Err((None, ChannelError::Close("Got commitment signed message when channel was not in an operational state".to_owned())));
                }
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
@@ -3124,7 +3131,7 @@ impl<Signer: Sign> Channel<Signer> {
        /// If we're not in a state where freeing the holding cell makes sense, this is a no-op and
        /// returns `(None, Vec::new())`.
        pub fn maybe_free_holding_cell_htlcs<L: Deref>(&mut self, logger: &L) -> Result<(Option<(msgs::CommitmentUpdate, ChannelMonitorUpdate)>, Vec<(HTLCSource, PaymentHash)>), ChannelError> where L::Target: Logger {
-               if self.channel_state >= ChannelState::ChannelFunded as u32 &&
+               if self.channel_state >= ChannelState::ChannelReady as u32 &&
                   (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32 | ChannelState::PeerDisconnected as u32 | ChannelState::MonitorUpdateInProgress as u32)) == 0 {
                        self.free_holding_cell_htlcs(logger)
                } else { Ok((None, Vec::new())) }
@@ -3252,7 +3259,7 @@ impl<Signer: Sign> Channel<Signer> {
        pub fn revoke_and_ack<L: Deref>(&mut self, msg: &msgs::RevokeAndACK, logger: &L) -> Result<RAAUpdates, ChannelError>
                where L::Target: Logger,
        {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Close("Got revoke/ACK message when channel was not in an operational state".to_owned()));
                }
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
@@ -3696,7 +3703,7 @@ impl<Signer: Sign> Channel<Signer> {
                        } else { None };
                // That said, if the funding transaction is already confirmed (ie we're active with a
                // minimum_depth over 0) don't bother re-broadcasting the confirmed funding tx.
-               if self.channel_state & !MULTI_STATE_FLAGS >= ChannelState::ChannelFunded as u32 && self.minimum_depth != Some(0) {
+               if self.channel_state & !MULTI_STATE_FLAGS >= ChannelState::ChannelReady as u32 && self.minimum_depth != Some(0) {
                        funding_broadcastable = None;
                }
 
@@ -4198,7 +4205,7 @@ impl<Signer: Sign> Channel<Signer> {
        pub fn shutdown<K: Deref>(
                &mut self, keys_provider: &K, their_features: &InitFeatures, msg: &msgs::Shutdown
        ) -> Result<(Option<msgs::Shutdown>, Option<ChannelMonitorUpdate>, Vec<(HTLCSource, PaymentHash)>), ChannelError>
-       where K::Target: KeysInterface<Signer = Signer>
+       where K::Target: KeysInterface
        {
                if self.channel_state & (ChannelState::PeerDisconnected as u32) == ChannelState::PeerDisconnected as u32 {
                        return Err(ChannelError::Close("Peer sent shutdown when we needed a channel_reestablish".to_owned()));
@@ -4513,6 +4520,11 @@ impl<Signer: Sign> Channel<Signer> {
                self.channel_transaction_parameters.funding_outpoint
        }
 
+       /// Returns the block hash in which our funding transaction was confirmed.
+       pub fn get_funding_tx_confirmed_in(&self) -> Option<BlockHash> {
+               self.funding_tx_confirmed_in
+       }
+
        fn get_holder_selected_contest_delay(&self) -> u16 {
                self.channel_transaction_parameters.holder_selected_contest_delay
        }
@@ -4598,6 +4610,16 @@ impl<Signer: Sign> Channel<Signer> {
                self.prev_config.map(|prev_config| prev_config.0)
        }
 
+       // Checks whether we should emit a `ChannelReady` event.
+       pub(crate) fn should_emit_channel_ready_event(&mut self) -> bool {
+               self.is_usable() && !self.channel_ready_event_emitted
+       }
+
+       // Remembers that we already emitted a `ChannelReady` event.
+       pub(crate) fn set_channel_ready_event_emitted(&mut self) {
+               self.channel_ready_event_emitted = true;
+       }
+
        /// Tracks the number of ticks elapsed since the previous [`ChannelConfig`] was updated. Once
        /// [`EXPIRE_PREV_CONFIG_TICKS`] is reached, the previous config is considered expired and will
        /// no longer be considered when forwarding HTLCs.
@@ -4766,8 +4788,8 @@ impl<Signer: Sign> Channel<Signer> {
        /// Returns true if this channel is fully established and not known to be closing.
        /// Allowed in any state (including after shutdown)
        pub fn is_usable(&self) -> bool {
-               let mask = ChannelState::ChannelFunded as u32 | BOTH_SIDES_SHUTDOWN_MASK;
-               (self.channel_state & mask) == (ChannelState::ChannelFunded as u32) && !self.monitor_pending_channel_ready
+               let mask = ChannelState::ChannelReady as u32 | BOTH_SIDES_SHUTDOWN_MASK;
+               (self.channel_state & mask) == (ChannelState::ChannelReady as u32) && !self.monitor_pending_channel_ready
        }
 
        /// Returns true if this channel is currently available for use. This is a superset of
@@ -4826,7 +4848,7 @@ impl<Signer: Sign> Channel<Signer> {
 
        /// Returns true if our channel_ready has been sent
        pub fn is_our_channel_ready(&self) -> bool {
-               (self.channel_state & ChannelState::OurChannelReady as u32) != 0 || self.channel_state >= ChannelState::ChannelFunded as u32
+               (self.channel_state & ChannelState::OurChannelReady as u32) != 0 || self.channel_state >= ChannelState::ChannelReady as u32
        }
 
        /// Returns true if our peer has either initiated or agreed to shut down the channel.
@@ -4880,14 +4902,14 @@ impl<Signer: Sign> Channel<Signer> {
                        self.channel_state |= ChannelState::OurChannelReady as u32;
                        true
                } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::TheirChannelReady as u32) {
-                       self.channel_state = ChannelState::ChannelFunded as u32 | (self.channel_state & MULTI_STATE_FLAGS);
+                       self.channel_state = ChannelState::ChannelReady as u32 | (self.channel_state & MULTI_STATE_FLAGS);
                        self.update_time_counter += 1;
                        true
                } else if non_shutdown_state == (ChannelState::FundingSent as u32 | ChannelState::OurChannelReady as u32) {
                        // We got a reorg but not enough to trigger a force close, just ignore.
                        false
                } else {
-                       if self.funding_tx_confirmation_height != 0 && self.channel_state < ChannelState::ChannelFunded as u32 {
+                       if self.funding_tx_confirmation_height != 0 && self.channel_state < ChannelState::ChannelReady as u32 {
                                // We should never see a funding transaction on-chain until we've received
                                // funding_signed (if we're an outbound channel), or seen funding_generated (if we're
                                // an inbound channel - before that we have no known funding TXID). The fuzzer,
@@ -5031,7 +5053,7 @@ impl<Signer: Sign> Channel<Signer> {
                }
 
                let non_shutdown_state = self.channel_state & (!MULTI_STATE_FLAGS);
-               if non_shutdown_state >= ChannelState::ChannelFunded as u32 ||
+               if non_shutdown_state >= ChannelState::ChannelReady as u32 ||
                   (non_shutdown_state & ChannelState::OurChannelReady as u32) == ChannelState::OurChannelReady as u32 {
                        let mut funding_tx_confirmations = height as i64 - self.funding_tx_confirmation_height as i64 + 1;
                        if self.funding_tx_confirmation_height == 0 {
@@ -5059,7 +5081,7 @@ impl<Signer: Sign> Channel<Signer> {
                                height >= self.channel_creation_height + FUNDING_CONF_DEADLINE_BLOCKS {
                        log_info!(logger, "Closing channel {} due to funding timeout", log_bytes!(self.channel_id));
                        // If funding_tx_confirmed_in is unset, the channel must not be active
-                       assert!(non_shutdown_state <= ChannelState::ChannelFunded as u32);
+                       assert!(non_shutdown_state <= ChannelState::ChannelReady as u32);
                        assert_eq!(non_shutdown_state & ChannelState::OurChannelReady as u32, 0);
                        return Err(ClosureReason::FundingTimedOut);
                }
@@ -5484,7 +5506,7 @@ impl<Signer: Sign> Channel<Signer> {
        ///
        /// If an Err is returned, it's a ChannelError::Ignore!
        pub fn send_htlc<L: Deref>(&mut self, amount_msat: u64, payment_hash: PaymentHash, cltv_expiry: u32, source: HTLCSource, onion_routing_packet: msgs::OnionPacket, logger: &L) -> Result<Option<msgs::UpdateAddHTLC>, ChannelError> where L::Target: Logger {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32 | BOTH_SIDES_SHUTDOWN_MASK)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32 | BOTH_SIDES_SHUTDOWN_MASK)) != (ChannelState::ChannelReady as u32) {
                        return Err(ChannelError::Ignore("Cannot send HTLC until channel is fully established and we haven't started shutting down".to_owned()));
                }
                let channel_total_msat = self.channel_value_satoshis * 1000;
@@ -5617,7 +5639,7 @@ impl<Signer: Sign> Channel<Signer> {
        /// last call to this Channel) send_htlc returned Ok(Some(_)) and there is an Err.
        /// May panic if called except immediately after a successful, Ok(Some(_))-returning send_htlc.
        pub fn send_commitment<L: Deref>(&mut self, logger: &L) -> Result<(msgs::CommitmentSigned, ChannelMonitorUpdate), ChannelError> where L::Target: Logger {
-               if (self.channel_state & (ChannelState::ChannelFunded as u32)) != (ChannelState::ChannelFunded as u32) {
+               if (self.channel_state & (ChannelState::ChannelReady as u32)) != (ChannelState::ChannelReady as u32) {
                        panic!("Cannot create commitment tx until channel is fully established");
                }
                if (self.channel_state & (ChannelState::AwaitingRemoteRevoke as u32)) == (ChannelState::AwaitingRemoteRevoke as u32) {
@@ -5803,7 +5825,7 @@ impl<Signer: Sign> Channel<Signer> {
        /// holding cell HTLCs for payment failure.
        pub fn get_shutdown<K: Deref>(&mut self, keys_provider: &K, their_features: &InitFeatures, target_feerate_sats_per_kw: Option<u32>)
        -> Result<(msgs::Shutdown, Option<ChannelMonitorUpdate>, Vec<(HTLCSource, PaymentHash)>), APIError>
-       where K::Target: KeysInterface<Signer = Signer> {
+       where K::Target: KeysInterface {
                for htlc in self.pending_outbound_htlcs.iter() {
                        if let OutboundHTLCState::LocalAnnounced(_) = htlc.state {
                                return Err(APIError::APIMisuseError{err: "Cannot begin shutdown with pending HTLCs. Process pending events first".to_owned()});
@@ -5906,7 +5928,7 @@ impl<Signer: Sign> Channel<Signer> {
                        // funding transaction, don't return a funding txo (which prevents providing the
                        // monitor update to the user, even if we return one).
                        // See test_duplicate_chan_id and test_pre_lockin_no_chan_closed_update for more.
-                       if self.channel_state & (ChannelState::FundingSent as u32 | ChannelState::ChannelFunded as u32 | ChannelState::ShutdownComplete as u32) != 0 {
+                       if self.channel_state & (ChannelState::FundingSent as u32 | ChannelState::ChannelReady as u32 | ChannelState::ShutdownComplete as u32) != 0 {
                                self.latest_monitor_update_id += 1;
                                Some((funding_txo, ChannelMonitorUpdate {
                                        update_id: self.latest_monitor_update_id,
@@ -6230,6 +6252,8 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
                        if self.holder_max_htlc_value_in_flight_msat != Self::get_holder_max_htlc_value_in_flight_msat(self.channel_value_satoshis, &old_max_in_flight_percent_config)
                        { Some(self.holder_max_htlc_value_in_flight_msat) } else { None };
 
+               let channel_ready_event_emitted = Some(self.channel_ready_event_emitted);
+
                write_tlv_fields!(writer, {
                        (0, self.announcement_sigs, option),
                        // minimum_depth and counterparty_selected_channel_reserve_satoshis used to have a
@@ -6252,6 +6276,7 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
                        (17, self.announcement_sigs_state, required),
                        (19, self.latest_inbound_scid_alias, option),
                        (21, self.outbound_scid_alias, required),
+                       (23, channel_ready_event_emitted, option),
                });
 
                Ok(())
@@ -6259,8 +6284,8 @@ impl<Signer: Sign> Writeable for Channel<Signer> {
 }
 
 const MAX_ALLOC_SIZE: usize = 64*1024;
-impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<Signer>
-               where K::Target: KeysInterface<Signer = Signer> {
+impl<'a, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<<K::Target as KeysInterface>::Signer>
+               where K::Target: KeysInterface {
        fn read<R : io::Read>(reader: &mut R, args: (&'a K, u32)) -> Result<Self, DecodeError> {
                let (keys_source, serialized_height) = args;
                let ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
@@ -6509,6 +6534,7 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<Signer>
                let mut announcement_sigs_state = Some(AnnouncementSigsState::NotSent);
                let mut latest_inbound_scid_alias = None;
                let mut outbound_scid_alias = None;
+               let mut channel_ready_event_emitted = None;
 
                read_tlv_fields!(reader, {
                        (0, announcement_sigs, option),
@@ -6526,6 +6552,7 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<Signer>
                        (17, announcement_sigs_state, option),
                        (19, latest_inbound_scid_alias, option),
                        (21, outbound_scid_alias, option),
+                       (23, channel_ready_event_emitted, option),
                });
 
                if let Some(preimages) = preimages_opt {
@@ -6666,6 +6693,8 @@ impl<'a, Signer: Sign, K: Deref> ReadableArgs<(&'a K, u32)> for Channel<Signer>
                        // Later in the ChannelManager deserialization phase we scan for channels and assign scid aliases if its missing
                        outbound_scid_alias: outbound_scid_alias.unwrap_or(0),
 
+                       channel_ready_event_emitted: channel_ready_event_emitted.unwrap_or(true),
+
                        #[cfg(any(test, fuzzing))]
                        historical_inbound_htlc_fulfills,
 
index 8e279f78a3a6f4b72efc5500ac77dd266bbb5043..d3fbc244021b059a16d12509510647801ae93549 100644 (file)
@@ -51,9 +51,9 @@ use crate::ln::msgs;
 use crate::ln::onion_utils;
 use crate::ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT};
 use crate::ln::wire::Encode;
-use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, InMemorySigner, Recipient};
+use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, Recipient};
 use crate::util::config::{UserConfig, ChannelConfig};
-use crate::util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
+use crate::util::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
 use crate::util::{byte_utils, events};
 use crate::util::wakers::{Future, Notifier};
 use crate::util::scid_utils::fake_scid;
@@ -66,7 +66,7 @@ use crate::prelude::*;
 use core::{cmp, mem};
 use core::cell::RefCell;
 use crate::io::Read;
-use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard};
+use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, FairRwLock};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use core::time::Duration;
 use core::ops::Deref;
@@ -112,7 +112,8 @@ pub(super) struct PendingHTLCInfo {
        pub(super) routing: PendingHTLCRouting,
        pub(super) incoming_shared_secret: [u8; 32],
        payment_hash: PaymentHash,
-       pub(super) amt_to_forward: u64,
+       pub(super) incoming_amt_msat: Option<u64>, // Added in 0.0.113
+       pub(super) outgoing_amt_msat: u64,
        pub(super) outgoing_cltv_value: u32,
 }
 
@@ -129,20 +130,22 @@ pub(super) enum PendingHTLCStatus {
        Fail(HTLCFailureMsg),
 }
 
-pub(super) enum HTLCForwardInfo {
-       AddHTLC {
-               forward_info: PendingHTLCInfo,
+pub(super) struct PendingAddHTLCInfo {
+       pub(super) forward_info: PendingHTLCInfo,
 
-               // These fields are produced in `forward_htlcs()` and consumed in
-               // `process_pending_htlc_forwards()` for constructing the
-               // `HTLCSource::PreviousHopData` for failed and forwarded
-               // HTLCs.
-               //
-               // Note that this may be an outbound SCID alias for the associated channel.
-               prev_short_channel_id: u64,
-               prev_htlc_id: u64,
-               prev_funding_outpoint: OutPoint,
-       },
+       // These fields are produced in `forward_htlcs()` and consumed in
+       // `process_pending_htlc_forwards()` for constructing the
+       // `HTLCSource::PreviousHopData` for failed and forwarded
+       // HTLCs.
+       //
+       // Note that this may be an outbound SCID alias for the associated channel.
+       prev_short_channel_id: u64,
+       prev_htlc_id: u64,
+       prev_funding_outpoint: OutPoint,
+}
+
+pub(super) enum HTLCForwardInfo {
+       AddHTLC(PendingAddHTLCInfo),
        FailHTLC {
                htlc_id: u64,
                err_packet: msgs::OnionErrorPacket,
@@ -395,12 +398,6 @@ pub(super) enum RAACommitmentOrder {
 // Note this is only exposed in cfg(test):
 pub(super) struct ChannelHolder<Signer: Sign> {
        pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
-       /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
-       ///
-       /// Outbound SCID aliases are added here once the channel is available for normal use, with
-       /// SCIDs being added once the funding transaction is confirmed at the channel's required
-       /// confirmation depth.
-       pub(super) short_to_chan_info: HashMap<u64, (PublicKey, [u8; 32])>,
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
        /// failed/claimed by the user.
        ///
@@ -473,6 +470,7 @@ pub(crate) enum PendingOutboundPayment {
        Fulfilled {
                session_privs: HashSet<[u8; 32]>,
                payment_hash: Option<PaymentHash>,
+               timer_ticks_without_htlcs: u8,
        },
        /// When a payer gives up trying to retry a payment, they inform us, letting us generate a
        /// `PaymentFailed` event when all HTLCs have irrevocably failed. This avoids a number of race
@@ -488,12 +486,6 @@ pub(crate) enum PendingOutboundPayment {
 }
 
 impl PendingOutboundPayment {
-       fn is_retryable(&self) -> bool {
-               match self {
-                       PendingOutboundPayment::Retryable { .. } => true,
-                       _ => false,
-               }
-       }
        fn is_fulfilled(&self) -> bool {
                match self {
                        PendingOutboundPayment::Fulfilled { .. } => true,
@@ -532,7 +524,7 @@ impl PendingOutboundPayment {
                                => session_privs,
                });
                let payment_hash = self.payment_hash();
-               *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash };
+               *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash, timer_ticks_without_htlcs: 0 };
        }
 
        fn mark_abandoned(&mut self) -> Result<(), ()> {
@@ -617,7 +609,7 @@ impl PendingOutboundPayment {
 /// concrete type of the KeysManager.
 ///
 /// (C-not exported) as Arcs don't make sense in bindings
-pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<InMemorySigner, Arc<M>, Arc<T>, Arc<KeysManager>, Arc<F>, Arc<L>>;
+pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<Arc<M>, Arc<T>, Arc<KeysManager>, Arc<F>, Arc<L>>;
 
 /// SimpleRefChannelManager is a type alias for a ChannelManager reference, and is the reference
 /// counterpart to the SimpleArcChannelManager type alias. Use this type by default when you don't
@@ -629,7 +621,7 @@ pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<InMemorySigner, Ar
 /// concrete type of the KeysManager.
 ///
 /// (C-not exported) as Arcs don't make sense in bindings
-pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManager<InMemorySigner, &'a M, &'b T, &'c KeysManager, &'d F, &'e L>;
+pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManager<&'a M, &'b T, &'c KeysManager, &'d F, &'e L>;
 
 /// Manager which keeps track of a number of channels and sends messages to the appropriate
 /// channel, also tracking HTLC preimages and forwarding onion packets appropriately.
@@ -685,6 +677,8 @@ pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManage
 //  |   |
 //  |   |__`id_to_peer`
 //  |   |
+//  |   |__`short_to_chan_info`
+//  |   |
 //  |   |__`per_peer_state`
 //  |       |
 //  |       |__`outbound_scid_aliases`
@@ -699,10 +693,10 @@ pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManage
 //  |                   |
 //  |                   |__`pending_background_events`
 //
-pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
-       where M::Target: chain::Watch<Signer>,
+pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
                                L::Target: Logger,
 {
@@ -721,9 +715,9 @@ pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref,
 
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        #[cfg(any(test, feature = "_test_utils"))]
-       pub(super) channel_state: Mutex<ChannelHolder<Signer>>,
+       pub(super) channel_state: Mutex<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
        #[cfg(not(any(test, feature = "_test_utils")))]
-       channel_state: Mutex<ChannelHolder<Signer>>,
+       channel_state: Mutex<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
 
        /// Storage for PaymentSecrets and any requirements on future inbound payments before we will
        /// expose them to users via a PaymentReceived event. HTLCs which do not meet the requirements
@@ -791,6 +785,22 @@ pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref,
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        id_to_peer: Mutex<HashMap<[u8; 32], PublicKey>>,
 
+       /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
+       ///
+       /// Outbound SCID aliases are added here once the channel is available for normal use, with
+       /// SCIDs being added once the funding transaction is confirmed at the channel's required
+       /// confirmation depth.
+       ///
+       /// Note that while this holds `counterparty_node_id`s and `channel_id`s, no consistency
+       /// guarantees are made about the existence of a peer with the `counterparty_node_id` nor a
+       /// channel with the `channel_id` in our other maps.
+       ///
+       /// See `ChannelManager` struct-level documentation for lock order requirements.
+       #[cfg(test)]
+       pub(super) short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
+       #[cfg(not(test))]
+       short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
+
        our_network_key: SecretKey,
        our_network_pubkey: PublicKey,
 
@@ -959,13 +969,14 @@ const CHECK_CLTV_EXPIRY_SANITY: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_GRA
 #[allow(dead_code)]
 const CHECK_CLTV_EXPIRY_SANITY_2: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_GRACE_PERIOD_BLOCKS - 2*CLTV_CLAIM_BUFFER;
 
-/// The number of blocks before we consider an outbound payment for expiry if it doesn't have any
-/// pending HTLCs in flight.
-pub(crate) const PAYMENT_EXPIRY_BLOCKS: u32 = 3;
-
 /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until expiry of incomplete MPPs
 pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3;
 
+/// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the
+/// idempotency of payments by [`PaymentId`]. See
+/// [`ChannelManager::remove_stale_resolved_payments`].
+pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7;
+
 /// Information needed for constructing an invoice route hint for this channel.
 #[derive(Clone, Debug, PartialEq)]
 pub struct CounterpartyForwardingInfo {
@@ -1206,6 +1217,9 @@ pub enum PaymentSendFailure {
        /// All paths which were attempted failed to send, with no channel state change taking place.
        /// You can freely retry the payment in full (though you probably want to do so over different
        /// paths than the ones selected).
+       ///
+       /// [`ChannelManager::abandon_payment`] does *not* need to be called for this payment and
+       /// [`ChannelManager::retry_payment`] will *not* work for this payment.
        AllFailedRetrySafe(Vec<APIError>),
        /// Some paths which were attempted failed to send, though possibly not all. At least some
        /// paths have irrevocably committed to the HTLC and retrying the payment in full would result
@@ -1296,9 +1310,11 @@ macro_rules! handle_error {
 }
 
 macro_rules! update_maps_on_chan_removal {
-       ($self: expr, $short_to_chan_info: expr, $channel: expr) => {
+       ($self: expr, $channel: expr) => {{
+               $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
+               let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
                if let Some(short_id) = $channel.get_short_channel_id() {
-                       $short_to_chan_info.remove(&short_id);
+                       short_to_chan_info.remove(&short_id);
                } else {
                        // If the channel was never confirmed on-chain prior to its closure, remove the
                        // outbound SCID alias we used for it from the collision-prevention set. While we
@@ -1309,14 +1325,13 @@ macro_rules! update_maps_on_chan_removal {
                        let alias_removed = $self.outbound_scid_aliases.lock().unwrap().remove(&$channel.outbound_scid_alias());
                        debug_assert!(alias_removed);
                }
-               $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
-               $short_to_chan_info.remove(&$channel.outbound_scid_alias());
-       }
+               short_to_chan_info.remove(&$channel.outbound_scid_alias());
+       }}
 }
 
 /// Returns (boolean indicating if we should remove the Channel object from memory, a mapped error)
 macro_rules! convert_chan_err {
-       ($self: ident, $err: expr, $short_to_chan_info: expr, $channel: expr, $channel_id: expr) => {
+       ($self: ident, $err: expr, $channel: expr, $channel_id: expr) => {
                match $err {
                        ChannelError::Warn(msg) => {
                                (false, MsgHandleErrInternal::from_chan_no_close(ChannelError::Warn(msg), $channel_id.clone()))
@@ -1326,7 +1341,7 @@ macro_rules! convert_chan_err {
                        },
                        ChannelError::Close(msg) => {
                                log_error!($self.logger, "Closing channel {} due to close-required error: {}", log_bytes!($channel_id[..]), msg);
-                               update_maps_on_chan_removal!($self, $short_to_chan_info, $channel);
+                               update_maps_on_chan_removal!($self, $channel);
                                let shutdown_res = $channel.force_shutdown(true);
                                (true, MsgHandleErrInternal::from_finish_shutdown(msg, *$channel_id, $channel.get_user_id(),
                                        shutdown_res, $self.get_channel_update_for_broadcast(&$channel).ok()))
@@ -1336,11 +1351,11 @@ macro_rules! convert_chan_err {
 }
 
 macro_rules! break_chan_entry {
-       ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
+       ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
+                               let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
@@ -1351,11 +1366,11 @@ macro_rules! break_chan_entry {
 }
 
 macro_rules! try_chan_entry {
-       ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
+       ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
+                               let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
@@ -1366,21 +1381,21 @@ macro_rules! try_chan_entry {
 }
 
 macro_rules! remove_channel {
-       ($self: expr, $channel_state: expr, $entry: expr) => {
+       ($self: expr, $entry: expr) => {
                {
                        let channel = $entry.remove_entry().1;
-                       update_maps_on_chan_removal!($self, $channel_state.short_to_chan_info, channel);
+                       update_maps_on_chan_removal!($self, channel);
                        channel
                }
        }
 }
 
 macro_rules! handle_monitor_update_res {
-       ($self: ident, $err: expr, $short_to_chan_info: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
+       ($self: ident, $err: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
                match $err {
                        ChannelMonitorUpdateStatus::PermanentFailure => {
                                log_error!($self.logger, "Closing channel {} due to monitor update ChannelMonitorUpdateStatus::PermanentFailure", log_bytes!($chan_id[..]));
-                               update_maps_on_chan_removal!($self, $short_to_chan_info, $chan);
+                               update_maps_on_chan_removal!($self, $chan);
                                // TODO: $failed_fails is dropped here, which will cause other channels to hit the
                                // chain in a confused state! We need to move them into the ChannelMonitor which
                                // will be responsible for failing backwards once things confirm on-chain.
@@ -1422,47 +1437,65 @@ macro_rules! handle_monitor_update_res {
                        },
                }
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
-               let (res, drop) = handle_monitor_update_res!($self, $err, $channel_state.short_to_chan_info, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
+               let (res, drop) = handle_monitor_update_res!($self, $err, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
                if drop {
                        $entry.remove_entry();
                }
                res
        } };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
                debug_assert!($action_type == RAACommitmentOrder::CommitmentFirst);
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        } };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
        };
-       ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
-               handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
+       ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
+               handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
        };
 }
 
 macro_rules! send_channel_ready {
-       ($short_to_chan_info: expr, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {
+       ($self: ident, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {{
                $pending_msg_events.push(events::MessageSendEvent::SendChannelReady {
                        node_id: $channel.get_counterparty_node_id(),
                        msg: $channel_ready_msg,
                });
                // Note that we may send a `channel_ready` multiple times for a channel if we reconnect, so
                // we allow collisions, but we shouldn't ever be updating the channel ID pointed to.
-               let outbound_alias_insert = $short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
+               let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
+               let outbound_alias_insert = short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
                assert!(outbound_alias_insert.is_none() || outbound_alias_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                        "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                if let Some(real_scid) = $channel.get_short_channel_id() {
-                       let scid_insert = $short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
+                       let scid_insert = short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
                        assert!(scid_insert.is_none() || scid_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                                "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                }
+       }}
+}
+
+macro_rules! emit_channel_ready_event {
+       ($self: expr, $channel: expr) => {
+               if $channel.should_emit_channel_ready_event() {
+                       {
+                               let mut pending_events = $self.pending_events.lock().unwrap();
+                               pending_events.push(events::Event::ChannelReady {
+                                       channel_id: $channel.channel_id(),
+                                       user_channel_id: $channel.get_user_id(),
+                                       counterparty_node_id: $channel.get_counterparty_node_id(),
+                                       channel_type: $channel.get_channel_type().clone(),
+                               });
+                       }
+                       $channel.set_channel_ready_event_emitted();
+               }
        }
 }
 
@@ -1500,7 +1533,7 @@ macro_rules! handle_chan_restoration_locked {
                                // Similar to the above, this implies that we're letting the channel_ready fly
                                // before it should be allowed to.
                                assert!(chanmon_update.is_none());
-                               send_channel_ready!($channel_state.short_to_chan_info, $channel_state.pending_msg_events, $channel_entry.get(), msg);
+                               send_channel_ready!($self, $channel_state.pending_msg_events, $channel_entry.get(), msg);
                        }
                        if let Some(msg) = $announcement_sigs {
                                $channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
@@ -1509,6 +1542,8 @@ macro_rules! handle_chan_restoration_locked {
                                });
                        }
 
+                       emit_channel_ready_event!($self, $channel_entry.get_mut());
+
                        let funding_broadcastable: Option<Transaction> = $funding_broadcastable; // Force type-checking to resolve
                        if let Some(monitor_update) = chanmon_update {
                                // We only ever broadcast a funding transaction in response to a funding_signed
@@ -1531,7 +1566,7 @@ macro_rules! handle_chan_restoration_locked {
                                                if $raa.is_none() {
                                                        order = RAACommitmentOrder::CommitmentFirst;
                                                }
-                                               break handle_monitor_update_res!($self, e, $channel_state, $channel_entry, order, $raa.is_some(), true);
+                                               break handle_monitor_update_res!($self, e, $channel_entry, order, $raa.is_some(), true);
                                        }
                                }
                        }
@@ -1592,10 +1627,10 @@ macro_rules! post_handle_chan_restoration {
        } }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<Signer, M, T, K, F, L>
-       where M::Target: chain::Watch<Signer>,
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F, L>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
@@ -1625,7 +1660,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
-                               short_to_chan_info: HashMap::new(),
                                claimable_htlcs: HashMap::new(),
                                pending_msg_events: Vec::new(),
                        }),
@@ -1634,6 +1668,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        pending_outbound_payments: Mutex::new(HashMap::new()),
                        forward_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
+                       short_to_chan_info: FairRwLock::new(HashMap::new()),
 
                        our_network_key: keys_manager.get_node_secret(Recipient::Node).unwrap(),
                        our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &keys_manager.get_node_secret(Recipient::Node).unwrap()),
@@ -1761,7 +1796,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                Ok(temporary_channel_id)
        }
 
-       fn list_channels_with_filter<Fn: FnMut(&(&[u8; 32], &Channel<Signer>)) -> bool>(&self, f: Fn) -> Vec<ChannelDetails> {
+       fn list_channels_with_filter<Fn: FnMut(&(&[u8; 32], &Channel<<K::Target as KeysInterface>::Signer>)) -> bool>(&self, f: Fn) -> Vec<ChannelDetails> {
                let mut res = Vec::new();
                {
                        let channel_state = self.channel_state.lock().unwrap();
@@ -1843,7 +1878,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        }
 
        /// Helper function that issues the channel close events
-       fn issue_channel_close_events(&self, channel: &Channel<Signer>, closure_reason: ClosureReason) {
+       fn issue_channel_close_events(&self, channel: &Channel<<K::Target as KeysInterface>::Signer>, closure_reason: ClosureReason) {
                let mut pending_events_lock = self.pending_events.lock().unwrap();
                match channel.unbroadcasted_funding() {
                        Some(transaction) => {
@@ -1885,9 +1920,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
-                                                       handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
+                                                       handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
-                                                       remove_channel!(self, channel_state, chan_entry);
+                                                       remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
@@ -1898,7 +1933,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        });
 
                                        if chan_entry.get().is_shutdown() {
-                                               let channel = remove_channel!(self, channel_state, chan_entry);
+                                               let channel = remove_channel!(self, chan_entry);
                                                if let Ok(channel_update) = self.get_channel_update_for_broadcast(&channel) {
                                                        channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
                                                                msg: channel_update
@@ -1999,7 +2034,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                } else {
                                        self.issue_channel_close_events(chan.get(),ClosureReason::HolderForceClosed);
                                }
-                               remove_channel!(self, channel_state, chan)
+                               remove_channel!(self, chan)
                        } else {
                                return Err(APIError::ChannelUnavailable{err: "No such channel".to_owned()});
                        }
@@ -2103,13 +2138,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
 
                let routing = match hop_data.format {
-                       msgs::OnionHopDataFormat::Legacy { .. } => {
-                               return Err(ReceiveError {
-                                       err_code: 0x4000|0x2000|3,
-                                       err_data: Vec::new(),
-                                       msg: "We require payment_secrets",
-                               });
-                       },
                        msgs::OnionHopDataFormat::NonFinalNode { .. } => {
                                return Err(ReceiveError {
                                        err_code: 0x4000|22,
@@ -2162,7 +2190,8 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        routing,
                        payment_hash,
                        incoming_shared_secret: shared_secret,
-                       amt_to_forward: amt_msat,
+                       incoming_amt_msat: Some(amt_msat),
+                       outgoing_amt_msat: amt_msat,
                        outgoing_cltv_value: hop_data.outgoing_cltv_value,
                })
        }
@@ -2244,7 +2273,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                };
 
                                let short_channel_id = match next_hop_data.format {
-                                       msgs::OnionHopDataFormat::Legacy { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::NonFinalNode { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::FinalNode { .. } => {
                                                return_err!("Final Node OnionHopData provided for us as an intermediary node", 0x4000 | 22, &[0;0]);
@@ -2258,25 +2286,26 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        },
                                        payment_hash: msg.payment_hash.clone(),
                                        incoming_shared_secret: shared_secret,
-                                       amt_to_forward: next_hop_data.amt_to_forward,
+                                       incoming_amt_msat: Some(msg.amount_msat),
+                                       outgoing_amt_msat: next_hop_data.amt_to_forward,
                                        outgoing_cltv_value: next_hop_data.outgoing_cltv_value,
                                })
                        }
                };
 
-               if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref amt_to_forward, ref outgoing_cltv_value, .. }) = &pending_forward_info {
+               if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref outgoing_amt_msat, ref outgoing_cltv_value, .. }) = &pending_forward_info {
                        // If short_channel_id is 0 here, we'll reject the HTLC as there cannot be a channel
                        // with a short_channel_id of 0. This is important as various things later assume
                        // short_channel_id is non-0 in any ::Forward.
                        if let &PendingHTLCRouting::Forward { ref short_channel_id, .. } = routing {
                                if let Some((err, code, chan_update)) = loop {
+                                       let id_option = self.short_to_chan_info.read().unwrap().get(&short_channel_id).cloned();
                                        let mut channel_state = self.channel_state.lock().unwrap();
-                                       let id_option = channel_state.short_to_chan_info.get(&short_channel_id).cloned();
                                        let forwarding_id_opt = match id_option {
                                                None => { // unknown_next_peer
                                                        // Note that this is likely a timing oracle for detecting whether an scid is a
                                                        // phantom.
-                                                       if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id) {
+                                                       if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash) {
                                                                None
                                                        } else {
                                                                break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
@@ -2285,7 +2314,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                Some((_cp_id, chan_id)) => Some(chan_id.clone()),
                                        };
                                        let chan_update_opt = if let Some(forwarding_id) = forwarding_id_opt {
-                                               let chan = channel_state.by_id.get_mut(&forwarding_id).unwrap();
+                                               let chan = match channel_state.by_id.get_mut(&forwarding_id) {
+                                                       None => {
+                                                               // Channel was removed. The short_to_chan_info and by_id maps have
+                                                               // no consistency guarantees.
+                                                               break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
+                                                       },
+                                                       Some(chan) => chan
+                                               };
                                                if !chan.should_announce() && !self.default_configuration.accept_forwards_to_priv_channels {
                                                        // Note that the behavior here should be identical to the above block - we
                                                        // should NOT reveal the existence or non-existence of a private channel if
@@ -2308,10 +2344,10 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                if !chan.is_live() { // channel_disabled
                                                        break Some(("Forwarding channel is not in a ready state.", 0x1000 | 20, chan_update_opt));
                                                }
-                                               if *amt_to_forward < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
+                                               if *outgoing_amt_msat < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
                                                        break Some(("HTLC amount was below the htlc_minimum_msat", 0x1000 | 11, chan_update_opt));
                                                }
-                                               if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *amt_to_forward, *outgoing_cltv_value) {
+                                               if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *outgoing_amt_msat, *outgoing_cltv_value) {
                                                        break Some((err, code, chan_update_opt));
                                                }
                                                chan_update_opt
@@ -2379,7 +2415,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// [`MessageSendEvent::BroadcastChannelUpdate`] event.
        ///
        /// May be called with channel_state already locked!
-       fn get_channel_update_for_broadcast(&self, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
+       fn get_channel_update_for_broadcast(&self, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                if !chan.should_announce() {
                        return Err(LightningError {
                                err: "Cannot broadcast a channel_update for a private channel".to_owned(),
@@ -2398,7 +2434,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// and thus MUST NOT be called unless the recipient of the resulting message has already
        /// provided evidence that they know about the existence of the channel.
        /// May be called with channel_state already locked!
-       fn get_channel_update_for_unicast(&self, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
+       fn get_channel_update_for_unicast(&self, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                log_trace!(self.logger, "Attempting to generate channel update for channel {}", log_bytes!(chan.channel_id()));
                let short_channel_id = match chan.get_short_channel_id().or(chan.latest_inbound_scid_alias()) {
                        None => return Err(LightningError{err: "Channel not yet established".to_owned(), action: msgs::ErrorAction::IgnoreError}),
@@ -2407,7 +2443,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                self.get_channel_update_for_onion(short_channel_id, chan)
        }
-       fn get_channel_update_for_onion(&self, short_channel_id: u64, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
+       fn get_channel_update_for_onion(&self, short_channel_id: u64, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                log_trace!(self.logger, "Generating channel update for channel {}", log_bytes!(chan.channel_id()));
                let were_node_one = PublicKey::from_secret_key(&self.secp_ctx, &self.our_network_key).serialize()[..] < chan.get_counterparty_node_id().serialize()[..];
 
@@ -2434,10 +2470,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        }
 
        // Only public for testing, this should otherwise never be called direcly
-       pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_params: &Option<PaymentParameters>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>) -> Result<(), APIError> {
+       pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_params: &Option<PaymentParameters>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
                log_trace!(self.logger, "Attempting to send payment for path with next hop {}", path.first().unwrap().short_channel_id);
                let prng_seed = self.keys_manager.get_secure_random_bytes();
-               let session_priv_bytes = self.keys_manager.get_secure_random_bytes();
                let session_priv = SecretKey::from_slice(&session_priv_bytes[..]).expect("RNG is busted");
 
                let onion_keys = onion_utils::construct_onion_keys(&self.secp_ctx, &path, &session_priv)
@@ -2451,38 +2486,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
 
                let err: Result<(), _> = loop {
-                       let mut channel_lock = self.channel_state.lock().unwrap();
-
-                       let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
-                       let payment_entry = pending_outbounds.entry(payment_id);
-                       if let hash_map::Entry::Occupied(payment) = &payment_entry {
-                               if !payment.get().is_retryable() {
-                                       return Err(APIError::RouteError {
-                                               err: "Payment already completed"
-                                       });
-                               }
-                       }
-
-                       let id = match channel_lock.short_to_chan_info.get(&path.first().unwrap().short_channel_id) {
+                       let id = match self.short_to_chan_info.read().unwrap().get(&path.first().unwrap().short_channel_id) {
                                None => return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()}),
                                Some((_cp_id, chan_id)) => chan_id.clone(),
                        };
 
-                       macro_rules! insert_outbound_payment {
-                               () => {
-                                       let payment = payment_entry.or_insert_with(|| PendingOutboundPayment::Retryable {
-                                               session_privs: HashSet::new(),
-                                               pending_amt_msat: 0,
-                                               pending_fee_msat: Some(0),
-                                               payment_hash: *payment_hash,
-                                               payment_secret: *payment_secret,
-                                               starting_block_height: self.best_block.read().unwrap().height(),
-                                               total_msat: total_value,
-                                       });
-                                       assert!(payment.insert(session_priv_bytes, path));
-                               }
-                       }
-
+                       let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
                        if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(id) {
                                match {
@@ -2501,19 +2510,17 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        payment_secret: payment_secret.clone(),
                                                        payment_params: payment_params.clone(),
                                                }, onion_packet, &self.logger),
-                                       channel_state, chan)
+                                               chan)
                                } {
                                        Some((update_add, commitment_signed, monitor_update)) => {
                                                let update_err = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
                                                let chan_id = chan.get().channel_id();
                                                match (update_err,
-                                                       handle_monitor_update_res!(self, update_err, channel_state, chan,
+                                                       handle_monitor_update_res!(self, update_err, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false, true))
                                                {
                                                        (ChannelMonitorUpdateStatus::PermanentFailure, Err(e)) => break Err(e),
-                                                       (ChannelMonitorUpdateStatus::Completed, Ok(())) => {
-                                                               insert_outbound_payment!();
-                                                       },
+                                                       (ChannelMonitorUpdateStatus::Completed, Ok(())) => {},
                                                        (ChannelMonitorUpdateStatus::InProgress, Err(_)) => {
                                                                // Note that MonitorUpdateInProgress here indicates (per function
                                                                // docs) that we will resend the commitment update once monitor
@@ -2521,7 +2528,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                // indicating that it is unsafe to retry the payment wholesale,
                                                                // which we do in the send_payment check for
                                                                // MonitorUpdateInProgress, below.
-                                                               insert_outbound_payment!(); // Only do this after possibly break'ing on Perm failure above.
                                                                return Err(APIError::MonitorUpdateInProgress);
                                                        },
                                                        _ => unreachable!(),
@@ -2540,9 +2546,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        },
                                                });
                                        },
-                                       None => { insert_outbound_payment!(); },
+                                       None => { },
                                }
-                       } else { unreachable!(); }
+                       } else {
+                               // The channel was likely removed after we fetched the id from the
+                               // `short_to_chan_info` map, but before we successfully locked the `by_id` map.
+                               // This can occur as no consistency guarantees exists between the two maps.
+                               return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()});
+                       }
                        return Ok(());
                };
 
@@ -2559,14 +2570,20 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// Value parameters are provided via the last hop in route, see documentation for RouteHop
        /// fields for more info.
        ///
-       /// Note that if the payment_hash already exists elsewhere (eg you're sending a duplicative
-       /// payment), we don't do anything to stop you! We always try to ensure that if the provided
-       /// next hop knows the preimage to payment_hash they can claim an additional amount as
-       /// specified in the last hop in the route! Thus, you should probably do your own
-       /// payment_preimage tracking (which you should already be doing as they represent "proof of
-       /// payment") and prevent double-sends yourself.
+       /// If a pending payment is currently in-flight with the same [`PaymentId`] provided, this
+       /// method will error with an [`APIError::RouteError`]. Note, however, that once a payment
+       /// is no longer pending (either via [`ChannelManager::abandon_payment`], or handling of an
+       /// [`Event::PaymentSent`]) LDK will not stop you from sending a second payment with the same
+       /// [`PaymentId`].
        ///
-       /// May generate SendHTLCs message(s) event on success, which should be relayed.
+       /// Thus, in order to ensure duplicate payments are not sent, you should implement your own
+       /// tracking of payments, including state to indicate once a payment has completed. Because you
+       /// should also ensure that [`PaymentHash`]es are not re-used, for simplicity, you should
+       /// consider using the [`PaymentHash`] as the key for tracking payments. In that case, the
+       /// [`PaymentId`] should be a copy of the [`PaymentHash`] bytes.
+       ///
+       /// May generate SendHTLCs message(s) event on success, which should be relayed (e.g. via
+       /// [`PeerManager::process_events`]).
        ///
        /// Each path may have a different return value, and PaymentSendValue may return a Vec with
        /// each entry matching the corresponding-index entry in the route paths, see
@@ -2590,14 +2607,55 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// newer nodes, it will be provided to you in the invoice. If you do not have one, the Route
        /// must not contain multiple paths as multi-path payments require a recipient-provided
        /// payment_secret.
+       ///
        /// If a payment_secret *is* provided, we assume that the invoice had the payment_secret feature
        /// bit set (either as required or as available). If multiple paths are present in the Route,
        /// we assume the invoice had the basic_mpp feature set.
-       pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>) -> Result<PaymentId, PaymentSendFailure> {
-               self.send_payment_internal(route, payment_hash, payment_secret, None, None, None)
+       ///
+       /// [`Event::PaymentSent`]: events::Event::PaymentSent
+       /// [`PeerManager::process_events`]: crate::ln::peer_handler::PeerManager::process_events
+       pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, route)?;
+               self.send_payment_internal(route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs)
        }
 
-       fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: Option<PaymentId>, recv_value_msat: Option<u64>) -> Result<PaymentId, PaymentSendFailure> {
+       #[cfg(test)]
+       pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
+               self.add_new_pending_payment(payment_hash, payment_secret, payment_id, route)
+       }
+
+       fn add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
+               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
+               for _ in 0..route.paths.len() {
+                       onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
+               }
+
+               let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
+               match pending_outbounds.entry(payment_id) {
+                       hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::ParameterError(APIError::RouteError {
+                               err: "Payment already in progress"
+                       })),
+                       hash_map::Entry::Vacant(entry) => {
+                               let payment = entry.insert(PendingOutboundPayment::Retryable {
+                                       session_privs: HashSet::new(),
+                                       pending_amt_msat: 0,
+                                       pending_fee_msat: Some(0),
+                                       payment_hash,
+                                       payment_secret,
+                                       starting_block_height: self.best_block.read().unwrap().height(),
+                                       total_msat: route.get_total_amount(),
+                               });
+
+                               for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
+                                       assert!(payment.insert(*session_priv_bytes, path));
+                               }
+
+                               Ok(onion_session_privs)
+                       },
+               }
+       }
+
+       fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
                if route.paths.len() < 1 {
                        return Err(PaymentSendFailure::ParameterError(APIError::RouteError{err: "There must be at least one path to send over"}));
                }
@@ -2607,7 +2665,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let mut total_value = 0;
                let our_node_id = self.get_our_node_id();
                let mut path_errs = Vec::with_capacity(route.paths.len());
-               let payment_id = if let Some(id) = payment_id { id } else { PaymentId(self.keys_manager.get_secure_random_bytes()) };
                'path_check: for path in route.paths.iter() {
                        if path.len() < 1 || path.len() > 20 {
                                path_errs.push(Err(APIError::RouteError{err: "Path didn't go anywhere/had bogus size"}));
@@ -2632,8 +2689,28 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                let cur_height = self.best_block.read().unwrap().height() + 1;
                let mut results = Vec::new();
-               for path in route.paths.iter() {
-                       results.push(self.send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage));
+               debug_assert_eq!(route.paths.len(), onion_session_privs.len());
+               for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) {
+                       let mut path_res = self.send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv);
+                       match path_res {
+                               Ok(_) => {},
+                               Err(APIError::MonitorUpdateInProgress) => {
+                                       // While a MonitorUpdateInProgress is an Err(_), the payment is still
+                                       // considered "in flight" and we shouldn't remove it from the
+                                       // PendingOutboundPayment set.
+                               },
+                               Err(_) => {
+                                       let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
+                                       if let Some(payment) = pending_outbounds.get_mut(&payment_id) {
+                                               let removed = payment.remove(&session_priv, Some(path));
+                                               debug_assert!(removed, "This can't happen as the payment has an entry for this path added by callers");
+                                       } else {
+                                               debug_assert!(false, "This can't happen as the payment was added by callers");
+                                               path_res = Err(APIError::APIMisuseError { err: "Internal error: payment disappeared during processing. Please report this bug!".to_owned() });
+                                       }
+                               }
+                       }
+                       results.push(path_res);
                }
                let mut has_ok = false;
                let mut has_err = false;
@@ -2667,12 +2744,13 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                } else { None },
                        })
                } else if has_err {
-                       // If we failed to send any paths, we shouldn't have inserted the new PaymentId into
-                       // our `pending_outbound_payments` map at all.
-                       debug_assert!(self.pending_outbound_payments.lock().unwrap().get(&payment_id).is_none());
+                       // If we failed to send any paths, we should remove the new PaymentId from the
+                       // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`.
+                       let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some();
+                       debug_assert!(removed, "We should always have a pending payment to remove here");
                        Err(PaymentSendFailure::AllFailedRetrySafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
                } else {
-                       Ok(payment_id)
+                       Ok(())
                }
        }
 
@@ -2696,44 +2774,55 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        }
                }
 
+               let mut onion_session_privs = Vec::with_capacity(route.paths.len());
+               for _ in 0..route.paths.len() {
+                       onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
+               }
+
                let (total_msat, payment_hash, payment_secret) = {
-                       let outbounds = self.pending_outbound_payments.lock().unwrap();
-                       if let Some(payment) = outbounds.get(&payment_id) {
-                               match payment {
-                                       PendingOutboundPayment::Retryable {
-                                               total_msat, payment_hash, payment_secret, pending_amt_msat, ..
-                                       } => {
-                                               let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
-                                               if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
+                       let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+                       match outbounds.get_mut(&payment_id) {
+                               Some(payment) => {
+                                       let res = match payment {
+                                               PendingOutboundPayment::Retryable {
+                                                       total_msat, payment_hash, payment_secret, pending_amt_msat, ..
+                                               } => {
+                                                       let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
+                                                       if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
+                                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                                       err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
+                                                               }))
+                                                       }
+                                                       (*total_msat, *payment_hash, *payment_secret)
+                                               },
+                                               PendingOutboundPayment::Legacy { .. } => {
                                                        return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                               err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
+                                                               err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
                                                        }))
-                                               }
-                                               (*total_msat, *payment_hash, *payment_secret)
-                                       },
-                                       PendingOutboundPayment::Legacy { .. } => {
-                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                       err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
-                                               }))
-                                       },
-                                       PendingOutboundPayment::Fulfilled { .. } => {
-                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                       err: "Payment already completed".to_owned()
-                                               }));
-                                       },
-                                       PendingOutboundPayment::Abandoned { .. } => {
-                                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                                       err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
-                                               }));
-                                       },
-                               }
-                       } else {
-                               return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
-                                       err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
-                               }))
+                                               },
+                                               PendingOutboundPayment::Fulfilled { .. } => {
+                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                               err: "Payment already completed".to_owned()
+                                                       }));
+                                               },
+                                               PendingOutboundPayment::Abandoned { .. } => {
+                                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                                               err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
+                                                       }));
+                                               },
+                                       };
+                                       for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
+                                               assert!(payment.insert(*session_priv_bytes, path));
+                                       }
+                                       res
+                               },
+                               None =>
+                                       return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
+                                               err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
+                                       })),
                        }
                };
-               return self.send_payment_internal(route, payment_hash, &payment_secret, None, Some(payment_id), Some(total_msat)).map(|_| ())
+               self.send_payment_internal(route, payment_hash, &payment_secret, None, payment_id, Some(total_msat), onion_session_privs)
        }
 
        /// Signals that no further retries for the given payment will occur.
@@ -2773,7 +2862,8 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// would be able to guess -- otherwise, an intermediate node may claim the payment and it will
        /// never reach the recipient.
        ///
-       /// See [`send_payment`] documentation for more details on the return value of this function.
+       /// See [`send_payment`] documentation for more details on the return value of this function
+       /// and idempotency guarantees provided by the [`PaymentId`] key.
        ///
        /// Similar to regular payments, you MUST NOT reuse a `payment_preimage` value. See
        /// [`send_payment`] for more information about the risks of duplicate preimage usage.
@@ -2781,14 +2871,16 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// Note that `route` must have exactly one path.
        ///
        /// [`send_payment`]: Self::send_payment
-       pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>) -> Result<(PaymentHash, PaymentId), PaymentSendFailure> {
+       pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId) -> Result<PaymentHash, PaymentSendFailure> {
                let preimage = match payment_preimage {
                        Some(p) => p,
                        None => PaymentPreimage(self.keys_manager.get_secure_random_bytes()),
                };
                let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
-               match self.send_payment_internal(route, payment_hash, &None, Some(preimage), None, None) {
-                       Ok(payment_id) => Ok((payment_hash, payment_id)),
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
+
+               match self.send_payment_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs) {
+                       Ok(()) => Ok(payment_hash),
                        Err(e) => Err(e)
                }
        }
@@ -2808,9 +2900,10 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
 
                let route = Route { paths: vec![hops], payment_params: None };
+               let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
 
-               match self.send_payment_internal(&route, payment_hash, &None, None, Some(payment_id), None) {
-                       Ok(payment_id) => Ok((payment_hash, payment_id)),
+               match self.send_payment_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs) {
+                       Ok(()) => Ok((payment_hash, payment_id)),
                        Err(e) => Err(e)
                }
        }
@@ -2832,7 +2925,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
        /// Handles the generation of a funding transaction, optionally (for tests) with a function
        /// which checks the correctness of the funding transaction given the associated channel.
-       fn funding_transaction_generated_intern<FundingOutput: Fn(&Channel<Signer>, &Transaction) -> Result<OutPoint, APIError>>(
+       fn funding_transaction_generated_intern<FundingOutput: Fn(&Channel<<K::Target as KeysInterface>::Signer>, &Transaction) -> Result<OutPoint, APIError>>(
                &self, temporary_channel_id: &[u8; 32], _counterparty_node_id: &PublicKey, funding_transaction: Transaction, find_funding_output: FundingOutput
        ) -> Result<(), APIError> {
                let (chan, msg) = {
@@ -3049,87 +3142,90 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                if short_chan_id != 0 {
-                                       let forward_chan_id = match channel_state.short_to_chan_info.get(&short_chan_id) {
-                                               Some((_cp_id, chan_id)) => chan_id.clone(),
-                                               None => {
+                                       macro_rules! forwarding_channel_not_found {
+                                               () => {
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
-                                                                       HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                               routing, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
-                                                                               prev_funding_outpoint } => {
-                                                                                       macro_rules! failure_handler {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
-                                                                                                       log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
-
-                                                                                                       let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
-                                                                                                               short_channel_id: prev_short_channel_id,
-                                                                                                               outpoint: prev_funding_outpoint,
-                                                                                                               htlc_id: prev_htlc_id,
-                                                                                                               incoming_packet_shared_secret: incoming_shared_secret,
-                                                                                                               phantom_shared_secret: $phantom_ss,
-                                                                                                       });
-
-                                                                                                       let reason = if $next_hop_unknown {
-                                                                                                               HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
-                                                                                                       } else {
-                                                                                                               HTLCDestination::FailedPayment{ payment_hash }
-                                                                                                       };
-
-                                                                                                       failed_forwards.push((htlc_source, payment_hash,
-                                                                                                               HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
-                                                                                                               reason
-                                                                                                       ));
-                                                                                                       continue;
-                                                                                               }
+                                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                                               forward_info: PendingHTLCInfo {
+                                                                                       routing, incoming_shared_secret, payment_hash, outgoing_amt_msat,
+                                                                                       outgoing_cltv_value, incoming_amt_msat: _
+                                                                               }
+                                                                       }) => {
+                                                                               macro_rules! failure_handler {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
+                                                                                               log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
+
+                                                                                               let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                                                                                       short_channel_id: prev_short_channel_id,
+                                                                                                       outpoint: prev_funding_outpoint,
+                                                                                                       htlc_id: prev_htlc_id,
+                                                                                                       incoming_packet_shared_secret: incoming_shared_secret,
+                                                                                                       phantom_shared_secret: $phantom_ss,
+                                                                                               });
+
+                                                                                               let reason = if $next_hop_unknown {
+                                                                                                       HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
+                                                                                               } else {
+                                                                                                       HTLCDestination::FailedPayment{ payment_hash }
+                                                                                               };
+
+                                                                                               failed_forwards.push((htlc_source, payment_hash,
+                                                                                                       HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
+                                                                                                       reason
+                                                                                               ));
+                                                                                               continue;
                                                                                        }
-                                                                                       macro_rules! fail_forward {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
-                                                                                                       {
-                                                                                                               failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
-                                                                                                       }
+                                                                               }
+                                                                               macro_rules! fail_forward {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
+                                                                                               {
+                                                                                                       failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
                                                                                                }
                                                                                        }
-                                                                                       macro_rules! failed_payment {
-                                                                                               ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
-                                                                                                       {
-                                                                                                               failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
-                                                                                                       }
+                                                                               }
+                                                                               macro_rules! failed_payment {
+                                                                                       ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
+                                                                                               {
+                                                                                                       failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
                                                                                                }
                                                                                        }
-                                                                                       if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
-                                                                                               let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
-                                                                                               if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id) {
-                                                                                                       let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
-                                                                                                       let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
-                                                                                                               Ok(res) => res,
-                                                                                                               Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
-                                                                                                                       let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
-                                                                                                                       // In this scenario, the phantom would have sent us an
-                                                                                                                       // `update_fail_malformed_htlc`, meaning here we encrypt the error as
-                                                                                                                       // if it came from us (the second-to-last hop) but contains the sha256
-                                                                                                                       // of the onion.
-                                                                                                                       failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
-                                                                                                               },
-                                                                                                               Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
-                                                                                                                       failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
-                                                                                                               },
-                                                                                                       };
-                                                                                                       match next_hop {
-                                                                                                               onion_utils::Hop::Receive(hop_data) => {
-                                                                                                                       match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) {
-                                                                                                                               Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
-                                                                                                                               Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
-                                                                                                                       }
-                                                                                                               },
-                                                                                                               _ => panic!(),
-                                                                                                       }
-                                                                                               } else {
-                                                                                                       fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
+                                                                               }
+                                                                               if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
+                                                                                       let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
+                                                                                       if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id, &self.genesis_hash) {
+                                                                                               let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
+                                                                                               let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
+                                                                                                       Ok(res) => res,
+                                                                                                       Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
+                                                                                                               let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
+                                                                                                               // In this scenario, the phantom would have sent us an
+                                                                                                               // `update_fail_malformed_htlc`, meaning here we encrypt the error as
+                                                                                                               // if it came from us (the second-to-last hop) but contains the sha256
+                                                                                                               // of the onion.
+                                                                                                               failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
+                                                                                                       },
+                                                                                                       Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
+                                                                                                               failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
+                                                                                                       },
+                                                                                               };
+                                                                                               match next_hop {
+                                                                                                       onion_utils::Hop::Receive(hop_data) => {
+                                                                                                               match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value, Some(phantom_shared_secret)) {
+                                                                                                                       Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
+                                                                                                                       Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
+                                                                                                               }
+                                                                                                       },
+                                                                                                       _ => panic!(),
                                                                                                }
                                                                                        } else {
                                                                                                fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
                                                                                        }
-                                                                               },
+                                                                               } else {
+                                                                                       fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
+                                                                               }
+                                                                       },
                                                                        HTLCForwardInfo::FailHTLC { .. } => {
                                                                                // Channel went away before we could fail it. This implies
                                                                                // the channel is now on chain and our counterparty is
@@ -3138,143 +3234,158 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                        }
                                                                }
                                                        }
+                                               }
+                                       }
+                                       let forward_chan_id = match self.short_to_chan_info.read().unwrap().get(&short_chan_id) {
+                                               Some((_cp_id, chan_id)) => chan_id.clone(),
+                                               None => {
+                                                       forwarding_channel_not_found!();
                                                        continue;
                                                }
                                        };
-                                       if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(forward_chan_id) {
-                                               let mut add_htlc_msgs = Vec::new();
-                                               let mut fail_htlc_msgs = Vec::new();
-                                               for forward_info in pending_forwards.drain(..) {
-                                                       match forward_info {
-                                                               HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                               routing: PendingHTLCRouting::Forward {
-                                                                                       onion_packet, ..
-                                                                               }, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
-                                                                               prev_funding_outpoint } => {
-                                                                       log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
-                                                                       let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
-                                                                               short_channel_id: prev_short_channel_id,
-                                                                               outpoint: prev_funding_outpoint,
-                                                                               htlc_id: prev_htlc_id,
-                                                                               incoming_packet_shared_secret: incoming_shared_secret,
-                                                                               // Phantom payments are only PendingHTLCRouting::Receive.
-                                                                               phantom_shared_secret: None,
-                                                                       });
-                                                                       match chan.get_mut().send_htlc(amt_to_forward, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
-                                                                               Err(e) => {
-                                                                                       if let ChannelError::Ignore(msg) = e {
-                                                                                               log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
-                                                                                       } else {
-                                                                                               panic!("Stated return value requirements in send_htlc() were not met");
-                                                                                       }
-                                                                                       let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
-                                                                                       failed_forwards.push((htlc_source, payment_hash,
-                                                                                               HTLCFailReason::Reason { failure_code, data },
-                                                                                               HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
-                                                                                       ));
-                                                                                       continue;
+                                       match channel_state.by_id.entry(forward_chan_id) {
+                                               hash_map::Entry::Vacant(_) => {
+                                                       forwarding_channel_not_found!();
+                                                       continue;
+                                               },
+                                               hash_map::Entry::Occupied(mut chan) => {
+                                                       let mut add_htlc_msgs = Vec::new();
+                                                       let mut fail_htlc_msgs = Vec::new();
+                                                       for forward_info in pending_forwards.drain(..) {
+                                                               match forward_info {
+                                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint ,
+                                                                               forward_info: PendingHTLCInfo {
+                                                                                       incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value,
+                                                                                       routing: PendingHTLCRouting::Forward { onion_packet, .. }, incoming_amt_msat: _,
                                                                                },
-                                                                               Ok(update_add) => {
-                                                                                       match update_add {
-                                                                                               Some(msg) => { add_htlc_msgs.push(msg); },
-                                                                                               None => {
-                                                                                                       // Nothing to do here...we're waiting on a remote
-                                                                                                       // revoke_and_ack before we can add anymore HTLCs. The Channel
-                                                                                                       // will automatically handle building the update_add_htlc and
-                                                                                                       // commitment_signed messages when we can.
-                                                                                                       // TODO: Do some kind of timer to set the channel as !is_live()
-                                                                                                       // as we don't really want others relying on us relaying through
-                                                                                                       // this channel currently :/.
+                                                                       }) => {
+                                                                               log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
+                                                                               let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
+                                                                                       short_channel_id: prev_short_channel_id,
+                                                                                       outpoint: prev_funding_outpoint,
+                                                                                       htlc_id: prev_htlc_id,
+                                                                                       incoming_packet_shared_secret: incoming_shared_secret,
+                                                                                       // Phantom payments are only PendingHTLCRouting::Receive.
+                                                                                       phantom_shared_secret: None,
+                                                                               });
+                                                                               match chan.get_mut().send_htlc(outgoing_amt_msat, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
+                                                                                       Err(e) => {
+                                                                                               if let ChannelError::Ignore(msg) = e {
+                                                                                                       log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
+                                                                                               } else {
+                                                                                                       panic!("Stated return value requirements in send_htlc() were not met");
+                                                                                               }
+                                                                                               let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
+                                                                                               failed_forwards.push((htlc_source, payment_hash,
+                                                                                                       HTLCFailReason::Reason { failure_code, data },
+                                                                                                       HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
+                                                                                               ));
+                                                                                               continue;
+                                                                                       },
+                                                                                       Ok(update_add) => {
+                                                                                               match update_add {
+                                                                                                       Some(msg) => { add_htlc_msgs.push(msg); },
+                                                                                                       None => {
+                                                                                                               // Nothing to do here...we're waiting on a remote
+                                                                                                               // revoke_and_ack before we can add anymore HTLCs. The Channel
+                                                                                                               // will automatically handle building the update_add_htlc and
+                                                                                                               // commitment_signed messages when we can.
+                                                                                                               // TODO: Do some kind of timer to set the channel as !is_live()
+                                                                                                               // as we don't really want others relying on us relaying through
+                                                                                                               // this channel currently :/.
+                                                                                                       }
                                                                                                }
                                                                                        }
                                                                                }
-                                                                       }
-                                                               },
-                                                               HTLCForwardInfo::AddHTLC { .. } => {
-                                                                       panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
-                                                               },
-                                                               HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
-                                                                       log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
-                                                                       match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
-                                                                               Err(e) => {
-                                                                                       if let ChannelError::Ignore(msg) = e {
-                                                                                               log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
-                                                                                       } else {
-                                                                                               panic!("Stated return value requirements in get_update_fail_htlc() were not met");
+                                                                       },
+                                                                       HTLCForwardInfo::AddHTLC { .. } => {
+                                                                               panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
+                                                                       },
+                                                                       HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
+                                                                               log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
+                                                                               match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
+                                                                                       Err(e) => {
+                                                                                               if let ChannelError::Ignore(msg) = e {
+                                                                                                       log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
+                                                                                               } else {
+                                                                                                       panic!("Stated return value requirements in get_update_fail_htlc() were not met");
+                                                                                               }
+                                                                                               // fail-backs are best-effort, we probably already have one
+                                                                                               // pending, and if not that's OK, if not, the channel is on
+                                                                                               // the chain and sending the HTLC-Timeout is their problem.
+                                                                                               continue;
+                                                                                       },
+                                                                                       Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
+                                                                                       Ok(None) => {
+                                                                                               // Nothing to do here...we're waiting on a remote
+                                                                                               // revoke_and_ack before we can update the commitment
+                                                                                               // transaction. The Channel will automatically handle
+                                                                                               // building the update_fail_htlc and commitment_signed
+                                                                                               // messages when we can.
+                                                                                               // We don't need any kind of timer here as they should fail
+                                                                                               // the channel onto the chain if they can't get our
+                                                                                               // update_fail_htlc in time, it's not our problem.
                                                                                        }
-                                                                                       // fail-backs are best-effort, we probably already have one
-                                                                                       // pending, and if not that's OK, if not, the channel is on
-                                                                                       // the chain and sending the HTLC-Timeout is their problem.
-                                                                                       continue;
-                                                                               },
-                                                                               Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
-                                                                               Ok(None) => {
-                                                                                       // Nothing to do here...we're waiting on a remote
-                                                                                       // revoke_and_ack before we can update the commitment
-                                                                                       // transaction. The Channel will automatically handle
-                                                                                       // building the update_fail_htlc and commitment_signed
-                                                                                       // messages when we can.
-                                                                                       // We don't need any kind of timer here as they should fail
-                                                                                       // the channel onto the chain if they can't get our
-                                                                                       // update_fail_htlc in time, it's not our problem.
                                                                                }
-                                                                       }
-                                                               },
+                                                                       },
+                                                               }
                                                        }
-                                               }
 
-                                               if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
-                                                       let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
-                                                               Ok(res) => res,
-                                                               Err(e) => {
-                                                                       // We surely failed send_commitment due to bad keys, in that case
-                                                                       // close channel and then send error message to peer.
-                                                                       let counterparty_node_id = chan.get().get_counterparty_node_id();
-                                                                       let err: Result<(), _>  = match e {
-                                                                               ChannelError::Ignore(_) | ChannelError::Warn(_) => {
-                                                                                       panic!("Stated return value requirements in send_commitment() were not met");
-                                                                               }
-                                                                               ChannelError::Close(msg) => {
-                                                                                       log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
-                                                                                       let mut channel = remove_channel!(self, channel_state, chan);
-                                                                                       // ChannelClosed event is generated by handle_error for us.
-                                                                                       Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
-                                                                               },
-                                                                       };
-                                                                       handle_errors.push((counterparty_node_id, err));
-                                                                       continue;
-                                                               }
-                                                       };
-                                                       match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
-                                                               ChannelMonitorUpdateStatus::Completed => {},
-                                                               e => {
-                                                                       handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
-                                                                       continue;
+                                                       if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
+                                                               let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
+                                                                       Ok(res) => res,
+                                                                       Err(e) => {
+                                                                               // We surely failed send_commitment due to bad keys, in that case
+                                                                               // close channel and then send error message to peer.
+                                                                               let counterparty_node_id = chan.get().get_counterparty_node_id();
+                                                                               let err: Result<(), _>  = match e {
+                                                                                       ChannelError::Ignore(_) | ChannelError::Warn(_) => {
+                                                                                               panic!("Stated return value requirements in send_commitment() were not met");
+                                                                                       }
+                                                                                       ChannelError::Close(msg) => {
+                                                                                               log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
+                                                                                               let mut channel = remove_channel!(self, chan);
+                                                                                               // ChannelClosed event is generated by handle_error for us.
+                                                                                               Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
+                                                                                       },
+                                                                               };
+                                                                               handle_errors.push((counterparty_node_id, err));
+                                                                               continue;
+                                                                       }
+                                                               };
+                                                               match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
+                                                                       ChannelMonitorUpdateStatus::Completed => {},
+                                                                       e => {
+                                                                               handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
+                                                                               continue;
+                                                                       }
                                                                }
+                                                               log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
+                                                                       add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
+                                                               channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
+                                                                       node_id: chan.get().get_counterparty_node_id(),
+                                                                       updates: msgs::CommitmentUpdate {
+                                                                               update_add_htlcs: add_htlc_msgs,
+                                                                               update_fulfill_htlcs: Vec::new(),
+                                                                               update_fail_htlcs: fail_htlc_msgs,
+                                                                               update_fail_malformed_htlcs: Vec::new(),
+                                                                               update_fee: None,
+                                                                               commitment_signed: commitment_msg,
+                                                                       },
+                                                               });
                                                        }
-                                                       log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
-                                                               add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
-                                                       channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
-                                                               node_id: chan.get().get_counterparty_node_id(),
-                                                               updates: msgs::CommitmentUpdate {
-                                                                       update_add_htlcs: add_htlc_msgs,
-                                                                       update_fulfill_htlcs: Vec::new(),
-                                                                       update_fail_htlcs: fail_htlc_msgs,
-                                                                       update_fail_malformed_htlcs: Vec::new(),
-                                                                       update_fee: None,
-                                                                       commitment_signed: commitment_msg,
-                                                               },
-                                                       });
                                                }
-                                       } else {
-                                               unreachable!();
                                        }
                                } else {
                                        for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
-                                                       HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
-                                                                       routing, incoming_shared_secret, payment_hash, amt_to_forward, .. },
-                                                                       prev_funding_outpoint } => {
+                                                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
+                                                               forward_info: PendingHTLCInfo {
+                                                                       routing, incoming_shared_secret, payment_hash, outgoing_amt_msat, ..
+                                                               }
+                                                       }) => {
                                                                let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
                                                                        PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
@@ -3294,9 +3405,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                                incoming_packet_shared_secret: incoming_shared_secret,
                                                                                phantom_shared_secret,
                                                                        },
-                                                                       value: amt_to_forward,
+                                                                       value: outgoing_amt_msat,
                                                                        timer_ticks: 0,
-                                                                       total_msat: if let Some(data) = &payment_data { data.total_msat } else { amt_to_forward },
+                                                                       total_msat: if let Some(data) = &payment_data { data.total_msat } else { outgoing_amt_msat },
                                                                        cltv_expiry,
                                                                        onion_payload,
                                                                };
@@ -3403,7 +3514,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                                                                e.insert((purpose.clone(), vec![claimable_htlc]));
                                                                                                                new_events.push(events::Event::PaymentReceived {
                                                                                                                        payment_hash,
-                                                                                                                       amount_msat: amt_to_forward,
+                                                                                                                       amount_msat: outgoing_amt_msat,
                                                                                                                        purpose,
                                                                                                                });
                                                                                                        },
@@ -3492,7 +3603,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                self.process_background_events();
        }
 
-       fn update_channel_fee(&self, short_to_chan_info: &mut HashMap<u64, (PublicKey, [u8; 32])>, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
+       fn update_channel_fee(&self, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
                if !chan.is_outbound() { return (true, NotifyOption::SkipPersist, Ok(())); }
                // If the feerate has decreased by less than half, don't bother
                if new_feerate <= chan.get_feerate() && new_feerate * 2 > chan.get_feerate() {
@@ -3512,7 +3623,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let res = match chan.send_update_fee_and_commit(new_feerate, &self.logger) {
                        Ok(res) => Ok(res),
                        Err(e) => {
-                               let (drop, res) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
+                               let (drop, res) = convert_chan_err!(self, e, chan, chan_id);
                                if drop { retain_channel = false; }
                                Err(res)
                        }
@@ -3535,7 +3646,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                Ok(())
                                        },
                                        e => {
-                                               let (res, drop) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
+                                               let (res, drop) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
                                                if drop { retain_channel = false; }
                                                res
                                        }
@@ -3563,9 +3674,8 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
-                               let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
+                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push(err);
@@ -3578,6 +3688,45 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                });
        }
 
+       fn remove_stale_resolved_payments(&self) {
+               // If an outbound payment was completed, and no pending HTLCs remain, we should remove it
+               // from the map. However, if we did that immediately when the last payment HTLC is claimed,
+               // this could race the user making a duplicate send_payment call and our idempotency
+               // guarantees would be violated. Instead, we wait a few timer ticks to do the actual
+               // removal. This should be more than sufficient to ensure the idempotency of any
+               // `send_payment` calls that were made at the same time the `PaymentSent` event was being
+               // processed.
+               let mut pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
+               let pending_events = self.pending_events.lock().unwrap();
+               pending_outbound_payments.retain(|payment_id, payment| {
+                       if let PendingOutboundPayment::Fulfilled { session_privs, timer_ticks_without_htlcs, .. } = payment {
+                               let mut no_remaining_entries = session_privs.is_empty();
+                               if no_remaining_entries {
+                                       for ev in pending_events.iter() {
+                                               match ev {
+                                                       events::Event::PaymentSent { payment_id: Some(ev_payment_id), .. } |
+                                                       events::Event::PaymentPathSuccessful { payment_id: ev_payment_id, .. } |
+                                                       events::Event::PaymentPathFailed { payment_id: Some(ev_payment_id), .. } => {
+                                                               if payment_id == ev_payment_id {
+                                                                       no_remaining_entries = false;
+                                                                       break;
+                                                               }
+                                                       },
+                                                       _ => {},
+                                               }
+                                       }
+                               }
+                               if no_remaining_entries {
+                                       *timer_ticks_without_htlcs += 1;
+                                       *timer_ticks_without_htlcs <= IDEMPOTENCY_TIMEOUT_TICKS
+                               } else {
+                                       *timer_ticks_without_htlcs = 0;
+                                       true
+                               }
+                       } else { true }
+               });
+       }
+
        /// Performs actions which should happen on startup and roughly once per minute thereafter.
        ///
        /// This currently includes:
@@ -3603,10 +3752,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
-                               let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
                                        let counterparty_node_id = chan.get_counterparty_node_id();
-                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
+                                       let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push((err, counterparty_node_id));
@@ -3614,7 +3762,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if !retain_channel { return false; }
 
                                        if let Err(e) = chan.timer_check_closing_negotiation_progress() {
-                                               let (needs_close, err) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
+                                               let (needs_close, err) = convert_chan_err!(self, e, chan, chan_id);
                                                handle_errors.push((Err(err), chan.get_counterparty_node_id()));
                                                if needs_close { return false; }
                                        }
@@ -3681,6 +3829,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        for (err, counterparty_node_id) in handle_errors.drain(..) {
                                let _ = handle_error!(self, err, counterparty_node_id);
                        }
+
+                       self.remove_stale_resolved_payments();
+
                        should_persist
                });
        }
@@ -3723,7 +3874,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        ///
        /// This is for failures on the channel on which the HTLC was *received*, not failures
        /// forwarding
-       fn get_htlc_inbound_temp_fail_err_and_data(&self, desired_err_code: u16, chan: &Channel<Signer>) -> (u16, Vec<u8>) {
+       fn get_htlc_inbound_temp_fail_err_and_data(&self, desired_err_code: u16, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> (u16, Vec<u8>) {
                // We can't be sure what SCID was used when relaying inbound towards us, so we have to
                // guess somewhat. If its a public channel, we figure best to just use the real SCID (as
                // we're not leaking that we have a channel with the counterparty), otherwise we try to use
@@ -3743,7 +3894,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
        /// Gets an HTLC onion failure code and error data for an `UPDATE` error, given the error code
        /// that we want to return and a channel.
-       fn get_htlc_temp_fail_err_and_data(&self, desired_err_code: u16, scid: u64, chan: &Channel<Signer>) -> (u16, Vec<u8>) {
+       fn get_htlc_temp_fail_err_and_data(&self, desired_err_code: u16, scid: u64, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> (u16, Vec<u8>) {
                debug_assert_eq!(desired_err_code & 0x1000, 0x1000);
                if let Ok(upd) = self.get_channel_update_for_onion(scid, chan) {
                        let mut enc = VecWriter(Vec::with_capacity(upd.serialized_length() + 6));
@@ -4029,10 +4180,19 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        for htlc in sources.iter() {
-                               if let None = channel_state.short_to_chan_info.get(&htlc.prev_hop.short_channel_id) {
+                               let chan_id = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
+                                       Some((_cp_id, chan_id)) => chan_id.clone(),
+                                       None => {
+                                               valid_mpp = false;
+                                               break;
+                                       }
+                               };
+
+                               if let None = channel_state.by_id.get(&chan_id) {
                                        valid_mpp = false;
                                        break;
                                }
+
                                if expected_amt_msat.is_some() && expected_amt_msat != Some(htlc.total_msat) {
                                        log_error!(self.logger, "Somehow ended up with an MPP payment with different total amounts - this should not be reachable!");
                                        debug_assert!(false);
@@ -4113,10 +4273,10 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
        }
 
-       fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
+       fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
                //TODO: Delay the claimed_funds relaying just like we do outbound relay!
                let channel_state = &mut **channel_state_lock;
-               let chan_id = match channel_state.short_to_chan_info.get(&prev_hop.short_channel_id) {
+               let chan_id = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                return ClaimFundsFromHop::PrevHopForceClosed
@@ -4135,7 +4295,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                        payment_preimage, e);
                                                                return ClaimFundsFromHop::MonitorUpdateFail(
                                                                        chan.get().get_counterparty_node_id(),
-                                                                       handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
+                                                                       handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
                                                                        Some(htlc_value_msat)
                                                                );
                                                        }
@@ -4170,14 +4330,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                },
                                        }
                                        let counterparty_node_id = chan.get().get_counterparty_node_id();
-                                       let (drop, res) = convert_chan_err!(self, e, channel_state.short_to_chan_info, chan.get_mut(), &chan_id);
+                                       let (drop, res) = convert_chan_err!(self, e, chan.get_mut(), &chan_id);
                                        if drop {
                                                chan.remove_entry();
                                        }
                                        return ClaimFundsFromHop::MonitorUpdateFail(counterparty_node_id, res, None);
                                },
                        }
-               } else { unreachable!(); }
+               } else { return ClaimFundsFromHop::PrevHopForceClosed }
        }
 
        fn finalize_claims(&self, mut sources: Vec<HTLCSource>) {
@@ -4198,15 +4358,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        }
                                                );
                                        }
-                                       if payment.get().remaining_parts() == 0 {
-                                               payment.remove();
-                                       }
                                }
                        }
                }
        }
 
-       fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
+       fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
                match source {
                        HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } => {
                                mem::drop(channel_state_lock);
@@ -4246,10 +4403,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                }
                                                        );
                                                }
-
-                                               if payment.get().remaining_parts() == 0 {
-                                                       payment.remove();
-                                               }
                                        }
                                } else {
                                        log_trace!(self.logger, "Received duplicative fulfill for HTLC with payment_preimage {}", log_bytes!(payment_preimage.0));
@@ -4432,7 +4585,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                }
                                        };
                                        channel_state.pending_msg_events.push(send_msg_err_event);
-                                       let _ = remove_channel!(self, channel_state, channel);
+                                       let _ = remove_channel!(self, channel);
                                        return Err(APIError::APIMisuseError { err: "Please use accept_inbound_channel_from_trusted_peer_0conf to accept channels with zero confirmations.".to_owned() });
                                }
 
@@ -4512,7 +4665,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
-                                       try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), channel_state, chan);
+                                       try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), chan);
                                        (chan.get().get_value_satoshis(), chan.get().get_funding_redeemscript().to_v0_p2wsh(), chan.get().get_user_id())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
@@ -4539,7 +4692,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
-                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), channel_state, chan), chan.remove())
+                                       (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), chan), chan.remove())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
                        }
@@ -4592,7 +4745,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        msg: funding_msg,
                                });
                                if let Some(msg) = channel_ready {
-                                       send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan, msg);
+                                       send_channel_ready!(self, channel_state.pending_msg_events, chan, msg);
                                }
                                e.insert(chan);
                        }
@@ -4612,12 +4765,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        }
                                        let (monitor, funding_tx, channel_ready) = match chan.get_mut().funding_signed(&msg, best_block, &self.logger) {
                                                Ok(update) => update,
-                                               Err(e) => try_chan_entry!(self, Err(e), channel_state, chan),
+                                               Err(e) => try_chan_entry!(self, Err(e), chan),
                                        };
                                        match self.chain_monitor.watch_channel(chan.get().get_funding_txo().unwrap(), monitor) {
                                                ChannelMonitorUpdateStatus::Completed => {},
                                                e => {
-                                                       let mut res = handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
+                                                       let mut res = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
                                                        if let Err(MsgHandleErrInternal { ref mut shutdown_finish, .. }) = res {
                                                                // We weren't able to watch the channel to begin with, so no updates should be made on
                                                                // it. Previously, full_stack_target found an (unreachable) panic when the
@@ -4630,7 +4783,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                },
                                        }
                                        if let Some(msg) = channel_ready {
-                                               send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan.get(), msg);
+                                               send_channel_ready!(self, channel_state.pending_msg_events, chan.get(), msg);
                                        }
                                        funding_tx
                                },
@@ -4651,7 +4804,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
                                let announcement_sigs_opt = try_chan_entry!(self, chan.get_mut().channel_ready(&msg, self.get_our_node_id(),
-                                       self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), channel_state, chan);
+                                       self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), chan);
                                if let Some(announcement_sigs) = announcement_sigs_opt {
                                        log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(chan.get().channel_id()));
                                        channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
@@ -4672,6 +4825,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                });
                                        }
                                }
+
+                               emit_channel_ready_event!(self, chan.get_mut());
+
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -4696,16 +4852,16 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        if chan_entry.get().sent_shutdown() { " after we initiated shutdown" } else { "" });
                                        }
 
-                                       let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), channel_state, chan_entry);
+                                       let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), chan_entry);
                                        dropped_htlcs = htlcs;
 
                                        // Update the monitor with the shutdown script if necessary.
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
-                                                       handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
+                                                       handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
-                                                       remove_channel!(self, channel_state, chan_entry);
+                                                       remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
@@ -4740,7 +4896,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if chan_entry.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
-                                       let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), channel_state, chan_entry);
+                                       let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), chan_entry);
                                        if let Some(msg) = closing_signed {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendClosingSigned {
                                                        node_id: counterparty_node_id.clone(),
@@ -4753,7 +4909,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                // also implies there are no pending HTLCs left on the channel, so we can
                                                // fully delete it from tracking (the channel monitor is still around to
                                                // watch for old state broadcasts)!
-                                               (tx, Some(remove_channel!(self, channel_state, chan_entry)))
+                                               (tx, Some(remove_channel!(self, chan_entry)))
                                        } else { (tx, None) }
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -4795,7 +4951,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
 
-                               let create_pending_htlc_status = |chan: &Channel<Signer>, pending_forward_info: PendingHTLCStatus, error_code: u16| {
+                               let create_pending_htlc_status = |chan: &Channel<<K::Target as KeysInterface>::Signer>, pending_forward_info: PendingHTLCStatus, error_code: u16| {
                                        // If the update_add is completely bogus, the call will Err and we will close,
                                        // but if we've sent a shutdown and they haven't acknowledged it yet, we just
                                        // want to reject the new HTLC and fail it backwards instead of forwarding.
@@ -4817,7 +4973,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                _ => pending_forward_info
                                        }
                                };
-                               try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -4833,7 +4989,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
-                                       try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), channel_state, chan)
+                                       try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), chan)
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                        }
@@ -4850,7 +5006,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -4867,9 +5023,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                }
                                if (msg.failure_code & 0x8000) == 0 {
                                        let chan_err: ChannelError = ChannelError::Close("Got update_fail_malformed_htlc with BADONION not set".to_owned());
-                                       try_chan_entry!(self, Err(chan_err), channel_state, chan);
+                                       try_chan_entry!(self, Err(chan_err), chan);
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), chan);
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
@@ -4886,17 +5042,17 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                }
                                let (revoke_and_ack, commitment_signed, monitor_update) =
                                        match chan.get_mut().commitment_signed(&msg, &self.logger) {
-                                               Err((None, e)) => try_chan_entry!(self, Err(e), channel_state, chan),
+                                               Err((None, e)) => try_chan_entry!(self, Err(e), chan),
                                                Err((Some(update), e)) => {
                                                        assert!(chan.get().is_awaiting_monitor_update());
                                                        let _ = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), update);
-                                                       try_chan_entry!(self, Err(e), channel_state, chan);
+                                                       try_chan_entry!(self, Err(e), chan);
                                                        unreachable!();
                                                },
                                                Ok(res) => res
                                        };
                                let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
-                               if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
+                               if let Err(e) = handle_monitor_update_res!(self, update_res, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
                                        return Err(e);
                                }
 
@@ -4939,12 +5095,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        PendingHTLCRouting::ReceiveKeysend { .. } => 0,
                                        }) {
                                                hash_map::Entry::Occupied(mut entry) => {
-                                                       entry.get_mut().push(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
-                                                                                                       prev_htlc_id, forward_info });
+                                                       entry.get_mut().push(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info }));
                                                },
                                                hash_map::Entry::Vacant(entry) => {
-                                                       entry.insert(vec!(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
-                                                                                                    prev_htlc_id, forward_info }));
+                                                       entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
+                                                               prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info })));
                                                }
                                        }
                                }
@@ -4973,7 +5129,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        }
                                        let was_paused_for_mon_update = chan.get().is_awaiting_monitor_update();
                                        let raa_updates = break_chan_entry!(self,
-                                               chan.get_mut().revoke_and_ack(&msg, &self.logger), channel_state, chan);
+                                               chan.get_mut().revoke_and_ack(&msg, &self.logger), chan);
                                        htlcs_to_fail = raa_updates.holding_cell_failed_htlcs;
                                        let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), raa_updates.monitor_update);
                                        if was_paused_for_mon_update {
@@ -4985,7 +5141,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                break Err(MsgHandleErrInternal::ignore_no_close("Existing pending monitor update prevented responses to RAA".to_owned()));
                                        }
                                        if update_res != ChannelMonitorUpdateStatus::Completed {
-                                               if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan,
+                                               if let Err(e) = handle_monitor_update_res!(self, update_res, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false,
                                                                raa_updates.commitment_update.is_some(), false,
                                                                raa_updates.accepted_htlcs, raa_updates.failed_htlcs,
@@ -5033,7 +5189,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
-                               try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), channel_state, chan);
+                               try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
@@ -5055,7 +5211,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                                channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelAnnouncement {
                                        msg: try_chan_entry!(self, chan.get_mut().announcement_signatures(
-                                               self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), channel_state, chan),
+                                               self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), chan),
                                        // Note that announcement_signatures fails if the channel cannot be announced,
                                        // so get_channel_update_for_broadcast will never fail by the time we get here.
                                        update_msg: self.get_channel_update_for_broadcast(chan.get()).unwrap(),
@@ -5068,15 +5224,15 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
        /// Returns ShouldPersist if anything changed, otherwise either SkipPersist or an Err.
        fn internal_channel_update(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelUpdate) -> Result<NotifyOption, MsgHandleErrInternal> {
-               let mut channel_state_lock = self.channel_state.lock().unwrap();
-               let channel_state = &mut *channel_state_lock;
-               let chan_id = match channel_state.short_to_chan_info.get(&msg.contents.short_channel_id) {
+               let chan_id = match self.short_to_chan_info.read().unwrap().get(&msg.contents.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                // It's not a local channel
                                return Ok(NotifyOption::SkipPersist)
                        }
                };
+               let mut channel_state_lock = self.channel_state.lock().unwrap();
+               let channel_state = &mut *channel_state_lock;
                match channel_state.by_id.entry(chan_id) {
                        hash_map::Entry::Occupied(mut chan) => {
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
@@ -5094,10 +5250,10 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        return Ok(NotifyOption::SkipPersist);
                                } else {
                                        log_debug!(self.logger, "Received channel_update for channel {}.", log_bytes!(chan_id));
-                                       try_chan_entry!(self, chan.get_mut().channel_update(&msg), channel_state, chan);
+                                       try_chan_entry!(self, chan.get_mut().channel_update(&msg), chan);
                                }
                        },
-                       hash_map::Entry::Vacant(_) => unreachable!()
+                       hash_map::Entry::Vacant(_) => return Ok(NotifyOption::SkipPersist)
                }
                Ok(NotifyOption::DoPersist)
        }
@@ -5119,7 +5275,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        // add-HTLCs on disconnect, we may be handed HTLCs to fail backwards here.
                                        let responses = try_chan_entry!(self, chan.get_mut().channel_reestablish(
                                                msg, &self.logger, self.our_network_pubkey.clone(), self.genesis_hash,
-                                               &*self.best_block.read().unwrap()), channel_state, chan);
+                                               &*self.best_block.read().unwrap()), chan);
                                        let mut channel_update = None;
                                        if let Some(msg) = responses.shutdown_msg {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendShutdown {
@@ -5183,7 +5339,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                let by_id = &mut channel_state.by_id;
                                                let pending_msg_events = &mut channel_state.pending_msg_events;
                                                if let hash_map::Entry::Occupied(chan_entry) = by_id.entry(funding_outpoint.to_channel_id()) {
-                                                       let mut chan = remove_channel!(self, channel_state, chan_entry);
+                                                       let mut chan = remove_channel!(self, chan_entry);
                                                        failed_channels.push(chan.force_shutdown(false));
                                                        if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
                                                                pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
@@ -5242,7 +5398,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
 
                        by_id.retain(|channel_id, chan| {
@@ -5265,7 +5420,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                },
                                                                e => {
                                                                        has_monitor_update = true;
-                                                                       let (res, close_channel) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
+                                                                       let (res, close_channel) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
                                                                        handle_errors.push((chan.get_counterparty_node_id(), res));
                                                                        if close_channel { return false; }
                                                                },
@@ -5274,7 +5429,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                true
                                        },
                                        Err(e) => {
-                                               let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
+                                               let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                // ChannelClosed event is generated by handle_error for us
                                                !close_channel
@@ -5305,7 +5460,6 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
 
                        by_id.retain(|channel_id, chan| {
@@ -5330,13 +5484,13 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
                                                        log_info!(self.logger, "Broadcasting {}", log_tx!(tx));
                                                        self.tx_broadcaster.broadcast_transaction(&tx);
-                                                       update_maps_on_chan_removal!(self, short_to_chan_info, chan);
+                                                       update_maps_on_chan_removal!(self, chan);
                                                        false
                                                } else { true }
                                        },
                                        Err(e) => {
                                                has_update = true;
-                                               let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
+                                               let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                !close_channel
                                        }
@@ -5526,14 +5680,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        ///
        /// [phantom node payments]: crate::chain::keysinterface::PhantomKeysManager
        pub fn get_phantom_scid(&self) -> u64 {
-               let mut channel_state = self.channel_state.lock().unwrap();
-               let best_block = self.best_block.read().unwrap();
+               let best_block_height = self.best_block.read().unwrap().height();
+               let short_to_chan_info = self.short_to_chan_info.read().unwrap();
                loop {
-                       let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block.height(), &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
+                       let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block_height, &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
                        // Ensure the generated scid doesn't conflict with a real channel.
-                       match channel_state.short_to_chan_info.entry(scid_candidate) {
-                               hash_map::Entry::Occupied(_) => continue,
-                               hash_map::Entry::Vacant(_) => return scid_candidate
+                       match short_to_chan_info.get(&scid_candidate) {
+                               Some(_) => continue,
+                               None => return scid_candidate
                        }
                }
        }
@@ -5552,7 +5706,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        #[cfg(any(test, fuzzing, feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                let events = core::cell::RefCell::new(Vec::new());
-               let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
+               let event_handler = |event: events::Event| events.borrow_mut().push(event);
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
@@ -5566,12 +5720,45 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        pub fn clear_pending_payments(&self) {
                self.pending_outbound_payments.lock().unwrap().clear()
        }
+
+       /// Processes any events asynchronously in the order they were generated since the last call
+       /// using the given event handler.
+       ///
+       /// See the trait-level documentation of [`EventsProvider`] for requirements.
+       pub async fn process_pending_events_async<Future: core::future::Future, H: Fn(Event) -> Future>(
+               &self, handler: H
+       ) {
+               // We'll acquire our total consistency lock until the returned future completes so that
+               // we can be sure no other persists happen while processing events.
+               let _read_guard = self.total_consistency_lock.read().unwrap();
+
+               let mut result = NotifyOption::SkipPersist;
+
+               // TODO: This behavior should be documented. It's unintuitive that we query
+               // ChannelMonitors when clearing other events.
+               if self.process_pending_monitor_events() {
+                       result = NotifyOption::DoPersist;
+               }
+
+               let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
+               if !pending_events.is_empty() {
+                       result = NotifyOption::DoPersist;
+               }
+
+               for event in pending_events {
+                       handler(event).await;
+               }
+
+               if result == NotifyOption::DoPersist {
+                       self.persistence_notifier.notify();
+               }
+       }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSendEventsProvider for ChannelManager<Signer, M, T, K, F, L>
-       where M::Target: chain::Watch<Signer>,
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSendEventsProvider for ChannelManager<M, T, K, F, L>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
                                L::Target: Logger,
 {
@@ -5607,11 +5794,11 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSend
        }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> EventsProvider for ChannelManager<Signer, M, T, K, F, L>
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> EventsProvider for ChannelManager<M, T, K, F, L>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -5629,13 +5816,13 @@ where
                                result = NotifyOption::DoPersist;
                        }
 
-                       let mut pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
+                       let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
                        if !pending_events.is_empty() {
                                result = NotifyOption::DoPersist;
                        }
 
-                       for event in pending_events.drain(..) {
-                               handler.handle_event(&event);
+                       for event in pending_events {
+                               handler.handle_event(event);
                        }
 
                        result
@@ -5643,11 +5830,11 @@ where
        }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Listen for ChannelManager<Signer, M, T, K, F, L>
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Listen for ChannelManager<M, T, K, F, L>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -5680,11 +5867,11 @@ where
        }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Confirm for ChannelManager<Signer, M, T, K, F, L>
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Confirm for ChannelManager<M, T, K, F, L>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
@@ -5742,29 +5929,14 @@ where
                payment_secrets.retain(|_, inbound_payment| {
                        inbound_payment.expiry_time > header.time as u64
                });
-
-               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
-               let mut pending_events = self.pending_events.lock().unwrap();
-               outbounds.retain(|payment_id, payment| {
-                       if payment.remaining_parts() != 0 { return true }
-                       if let PendingOutboundPayment::Retryable { starting_block_height, payment_hash, .. } = payment {
-                               if *starting_block_height + PAYMENT_EXPIRY_BLOCKS <= height {
-                                       log_info!(self.logger, "Timing out payment with id {} and hash {}", log_bytes!(payment_id.0), log_bytes!(payment_hash.0));
-                                       pending_events.push(events::Event::PaymentFailed {
-                                               payment_id: *payment_id, payment_hash: *payment_hash,
-                                       });
-                                       false
-                               } else { true }
-                       } else { true }
-               });
        }
 
-       fn get_relevant_txids(&self) -> Vec<Txid> {
+       fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let channel_state = self.channel_state.lock().unwrap();
-               let mut res = Vec::with_capacity(channel_state.short_to_chan_info.len());
+               let mut res = Vec::with_capacity(channel_state.by_id.len());
                for chan in channel_state.by_id.values() {
-                       if let Some(funding_txo) = chan.get_funding_txo() {
-                               res.push(funding_txo.txid);
+                       if let (Some(funding_txo), block_hash) = (chan.get_funding_txo(), chan.get_funding_tx_confirmed_in()) {
+                               res.push((funding_txo.txid, block_hash));
                        }
                }
                res
@@ -5782,18 +5954,18 @@ where
        }
 }
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<Signer, M, T, K, F, L>
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F, L>
 where
-       M::Target: chain::Watch<Signer>,
+       M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
-       K::Target: KeysInterface<Signer = Signer>,
+       K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
 {
        /// Calls a function which handles an on-chain event (blocks dis/connected, transactions
        /// un/confirmed, etc) on each channel, handling any resulting errors or messages generated by
        /// the function.
-       fn do_chain_event<FN: Fn(&mut Channel<Signer>) -> Result<(Option<msgs::ChannelReady>, Vec<(HTLCSource, PaymentHash)>, Option<msgs::AnnouncementSignatures>), ClosureReason>>
+       fn do_chain_event<FN: Fn(&mut Channel<<K::Target as KeysInterface>::Signer>) -> Result<(Option<msgs::ChannelReady>, Vec<(HTLCSource, PaymentHash)>, Option<msgs::AnnouncementSignatures>), ClosureReason>>
                        (&self, height_opt: Option<u32>, f: FN) {
                // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
                // during initialization prior to the chain_monitor being fully configured in some cases.
@@ -5804,7 +5976,6 @@ where
                {
                        let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
                        channel_state.by_id.retain(|_, channel| {
                                let res = f(channel);
@@ -5816,7 +5987,7 @@ where
                                                }, HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() }));
                                        }
                                        if let Some(channel_ready) = channel_ready_opt {
-                                               send_channel_ready!(short_to_chan_info, pending_msg_events, channel, channel_ready);
+                                               send_channel_ready!(self, pending_msg_events, channel, channel_ready);
                                                if channel.is_usable() {
                                                        log_trace!(self.logger, "Sending channel_ready with private initial channel_update for our counterparty on channel {}", log_bytes!(channel.channel_id()));
                                                        if let Ok(msg) = self.get_channel_update_for_unicast(channel) {
@@ -5829,6 +6000,9 @@ where
                                                        log_trace!(self.logger, "Sending channel_ready WITHOUT channel_update for {}", log_bytes!(channel.channel_id()));
                                                }
                                        }
+
+                                       emit_channel_ready_event!(self, channel);
+
                                        if let Some(announcement_sigs) = announcement_sigs {
                                                log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(channel.channel_id()));
                                                pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
@@ -5854,6 +6028,7 @@ where
                                                        // enforce option_scid_alias then), and if the funding tx is ever
                                                        // un-confirmed we force-close the channel, ensuring short_to_chan_info
                                                        // is always consistent.
+                                                       let mut short_to_chan_info = self.short_to_chan_info.write().unwrap();
                                                        let scid_insert = short_to_chan_info.insert(real_scid, (channel.get_counterparty_node_id(), channel.channel_id()));
                                                        assert!(scid_insert.is_none() || scid_insert.unwrap() == (channel.get_counterparty_node_id(), channel.channel_id()),
                                                                "SCIDs should never collide - ensure you weren't behind by a full {} blocks when creating channels",
@@ -5861,7 +6036,7 @@ where
                                                }
                                        }
                                } else if let Err(reason) = res {
-                                       update_maps_on_chan_removal!(self, short_to_chan_info, channel);
+                                       update_maps_on_chan_removal!(self, channel);
                                        // It looks like our counterparty went on-chain or funding transaction was
                                        // reorged out of the main chain. Close the channel.
                                        failed_channels.push(channel.force_shutdown(true));
@@ -5958,11 +6133,11 @@ where
        }
 }
 
-impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
-       ChannelMessageHandler for ChannelManager<Signer, M, T, K, F, L>
-       where M::Target: chain::Watch<Signer>,
+impl<M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
+       ChannelMessageHandler for ChannelManager<M, T, K, F, L>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
@@ -6064,14 +6239,13 @@ impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
-                       let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        log_debug!(self.logger, "Marking channels with {} disconnected and generating channel_updates. We believe we {} make future connections to this peer.",
                                log_pubkey!(counterparty_node_id), if no_connection_possible { "cannot" } else { "can" });
                        channel_state.by_id.retain(|_, chan| {
                                if chan.get_counterparty_node_id() == *counterparty_node_id {
                                        chan.remove_uncommitted_htlcs_and_mark_paused(&self.logger);
                                        if chan.is_shutdown() {
-                                               update_maps_on_chan_removal!(self, short_to_chan_info, chan);
+                                               update_maps_on_chan_removal!(self, chan);
                                                self.issue_channel_close_events(chan, ClosureReason::DisconnectedPeer);
                                                return false;
                                        } else {
@@ -6330,8 +6504,9 @@ impl_writeable_tlv_based!(PendingHTLCInfo, {
        (0, routing, required),
        (2, incoming_shared_secret, required),
        (4, payment_hash, required),
-       (6, amt_to_forward, required),
-       (8, outgoing_cltv_value, required)
+       (6, outgoing_amt_msat, required),
+       (8, outgoing_cltv_value, required),
+       (9, incoming_amt_msat, option),
 });
 
 
@@ -6530,7 +6705,7 @@ impl Writeable for HTLCSource {
                                        (1, payment_id_opt, option),
                                        (2, first_hop_htlc_msat, required),
                                        (3, payment_secret, option),
-                                       (4, path, vec_type),
+                                       (4, *path, vec_type),
                                        (5, payment_params, option),
                                 });
                        }
@@ -6553,18 +6728,20 @@ impl_writeable_tlv_based_enum!(HTLCFailReason,
        },
 ;);
 
+impl_writeable_tlv_based!(PendingAddHTLCInfo, {
+       (0, forward_info, required),
+       (2, prev_short_channel_id, required),
+       (4, prev_htlc_id, required),
+       (6, prev_funding_outpoint, required),
+});
+
 impl_writeable_tlv_based_enum!(HTLCForwardInfo,
-       (0, AddHTLC) => {
-               (0, forward_info, required),
-               (2, prev_short_channel_id, required),
-               (4, prev_htlc_id, required),
-               (6, prev_funding_outpoint, required),
-       },
        (1, FailHTLC) => {
                (0, htlc_id, required),
                (2, err_packet, required),
-       },
-;);
+       };
+       (0, AddHTLC)
+);
 
 impl_writeable_tlv_based!(PendingInboundPayment, {
        (0, payment_secret, required),
@@ -6581,6 +6758,7 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
        (1, Fulfilled) => {
                (0, session_privs, required),
                (1, payment_hash, option),
+               (3, timer_ticks_without_htlcs, (default_value, 0)),
        },
        (2, Retryable) => {
                (0, session_privs, required),
@@ -6597,10 +6775,10 @@ impl_writeable_tlv_based_enum_upgradable!(PendingOutboundPayment,
        },
 );
 
-impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelManager<Signer, M, T, K, F, L>
-       where M::Target: chain::Watch<Signer>,
+impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelManager<M, T, K, F, L>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
@@ -6775,10 +6953,10 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable f
 /// which you've already broadcasted the transaction.
 ///
 /// [`ChainMonitor`]: crate::chain::chainmonitor::ChainMonitor
-pub struct ChannelManagerReadArgs<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
-       where M::Target: chain::Watch<Signer>,
+pub struct ChannelManagerReadArgs<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
@@ -6821,14 +6999,14 @@ pub struct ChannelManagerReadArgs<'a, Signer: 'a + Sign, M: Deref, T: Deref, K:
        /// this struct.
        ///
        /// (C-not exported) because we have no HashMap bindings
-       pub channel_monitors: HashMap<OutPoint, &'a mut ChannelMonitor<Signer>>,
+       pub channel_monitors: HashMap<OutPoint, &'a mut ChannelMonitor<<K::Target as KeysInterface>::Signer>>,
 }
 
-impl<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
-               ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>
-       where M::Target: chain::Watch<Signer>,
+impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
+               ChannelManagerReadArgs<'a, M, T, K, F, L>
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
                T::Target: BroadcasterInterface,
-               K::Target: KeysInterface<Signer = Signer>,
+               K::Target: KeysInterface,
                F::Target: FeeEstimator,
                L::Target: Logger,
        {
@@ -6836,7 +7014,7 @@ impl<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
        /// HashMap for you. This is primarily useful for C bindings where it is not practical to
        /// populate a HashMap directly from C.
        pub fn new(keys_manager: K, fee_estimator: F, chain_monitor: M, tx_broadcaster: T, logger: L, default_config: UserConfig,
-                       mut channel_monitors: Vec<&'a mut ChannelMonitor<Signer>>) -> Self {
+                       mut channel_monitors: Vec<&'a mut ChannelMonitor<<K::Target as KeysInterface>::Signer>>) -> Self {
                Self {
                        keys_manager, fee_estimator, chain_monitor, tx_broadcaster, logger, default_config,
                        channel_monitors: channel_monitors.drain(..).map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect()
@@ -6846,29 +7024,29 @@ impl<'a, Signer: 'a + Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 
 // Implement ReadableArgs for an Arc'd ChannelManager to make it a bit easier to work with the
 // SipmleArcChannelManager type:
-impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
-       ReadableArgs<ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<Signer, M, T, K, F, L>>)
-       where M::Target: chain::Watch<Signer>,
+impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
+       ReadableArgs<ChannelManagerReadArgs<'a, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<M, T, K, F, L>>)
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
-       fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
-               let (blockhash, chan_manager) = <(BlockHash, ChannelManager<Signer, M, T, K, F, L>)>::read(reader, args)?;
+       fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, M, T, K, F, L>) -> Result<Self, DecodeError> {
+               let (blockhash, chan_manager) = <(BlockHash, ChannelManager<M, T, K, F, L>)>::read(reader, args)?;
                Ok((blockhash, Arc::new(chan_manager)))
        }
 }
 
-impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
-       ReadableArgs<ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>> for (BlockHash, ChannelManager<Signer, M, T, K, F, L>)
-       where M::Target: chain::Watch<Signer>,
+impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
+       ReadableArgs<ChannelManagerReadArgs<'a, M, T, K, F, L>> for (BlockHash, ChannelManager<M, T, K, F, L>)
+       where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
         T::Target: BroadcasterInterface,
-        K::Target: KeysInterface<Signer = Signer>,
+        K::Target: KeysInterface,
         F::Target: FeeEstimator,
         L::Target: Logger,
 {
-       fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
+       fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, M, T, K, F, L>) -> Result<Self, DecodeError> {
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
 
                let genesis_hash: BlockHash = Readable::read(reader)?;
@@ -6884,7 +7062,7 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                let mut short_to_chan_info = HashMap::with_capacity(cmp::min(channel_count as usize, 128));
                let mut channel_closures = Vec::new();
                for _ in 0..channel_count {
-                       let mut channel: Channel<Signer> = Channel::read(reader, (&args.keys_manager, best_block_height))?;
+                       let mut channel: Channel<<K::Target as KeysInterface>::Signer> = Channel::read(reader, (&args.keys_manager, best_block_height))?;
                        let funding_txo = channel.get_funding_txo().ok_or(DecodeError::InvalidValue)?;
                        funding_txo_set.insert(funding_txo.clone());
                        if let Some(ref mut monitor) = args.channel_monitors.get_mut(&funding_txo) {
@@ -7260,7 +7438,6 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
-                               short_to_chan_info,
                                claimable_htlcs,
                                pending_msg_events: Vec::new(),
                        }),
@@ -7271,6 +7448,7 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                        forward_htlcs: Mutex::new(forward_htlcs),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
+                       short_to_chan_info: FairRwLock::new(short_to_chan_info),
                        fake_scid_rand_bytes: fake_scid_rand_bytes.unwrap(),
 
                        probing_cookie_secret: probing_cookie_secret.unwrap(),
@@ -7413,18 +7591,22 @@ mod tests {
 
                // First, send a partial MPP payment.
                let (route, our_payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
+               let mut mpp_route = route.clone();
+               mpp_route.paths.push(mpp_route.paths[0].clone());
+
                let payment_id = PaymentId([42; 32]);
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
-               nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None).unwrap();
+               let session_privs = nodes[0].node.add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &mpp_route).unwrap();
+               nodes[0].node.send_payment_along_path(&mpp_route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                pass_along_path(&nodes[0], &[&nodes[1]], 200_000, our_payment_hash, Some(payment_secret), events.drain(..).next().unwrap(), false, None);
 
                // Next, send a keysend payment with the same payment_hash and make sure it fails.
-               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
+               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7447,7 +7629,7 @@ mod tests {
                expect_payment_failed!(nodes[0], our_payment_hash, true);
 
                // Send the second half of the original MPP payment.
-               nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None).unwrap();
+               nodes[0].node.send_payment_along_path(&mpp_route.paths[1], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7545,7 +7727,7 @@ mod tests {
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
-               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
+               nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7578,7 +7760,7 @@ mod tests {
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
-               let (payment_hash, _) = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
+               let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7588,7 +7770,7 @@ mod tests {
 
                // Next, attempt a regular payment and make sure it fails.
                let payment_secret = PaymentSecret([43; 32]);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7631,7 +7813,7 @@ mod tests {
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], channelmanager::provided_init_features(), channelmanager::provided_init_features());
                let route_params = RouteParameters {
                        payment_params: PaymentParameters::for_keysend(payee_pubkey),
-                       final_value_msat: 10000,
+                       final_value_msat: 10_000,
                        final_cltv_expiry_delta: 40,
                };
                let network_graph = nodes[0].network_graph;
@@ -7645,7 +7827,8 @@ mod tests {
 
                let test_preimage = PaymentPreimage([42; 32]);
                let mismatch_payment_hash = PaymentHash([43; 32]);
-               let _ = nodes[0].node.send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), None, None).unwrap();
+               let session_privs = nodes[0].node.add_new_pending_payment(mismatch_payment_hash, None, PaymentId(mismatch_payment_hash.0), &route).unwrap();
+               nodes[0].node.send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -7675,7 +7858,7 @@ mod tests {
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], channelmanager::provided_init_features(), channelmanager::provided_init_features());
                let route_params = RouteParameters {
                        payment_params: PaymentParameters::for_keysend(payee_pubkey),
-                       final_value_msat: 10000,
+                       final_value_msat: 10_000,
                        final_cltv_expiry_delta: 40,
                };
                let network_graph = nodes[0].network_graph;
@@ -7690,7 +7873,8 @@ mod tests {
                let test_preimage = PaymentPreimage([42; 32]);
                let test_secret = PaymentSecret([43; 32]);
                let payment_hash = PaymentHash(Sha256::hash(&test_preimage.0).into_inner());
-               let _ = nodes[0].node.send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), None, None).unwrap();
+               let session_privs = nodes[0].node.add_new_pending_payment(payment_hash, Some(test_secret), PaymentId(payment_hash.0), &route).unwrap();
+               nodes[0].node.send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), PaymentId(payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -7727,7 +7911,7 @@ mod tests {
                route.paths[1][0].short_channel_id = chan_2_id;
                route.paths[1][1].short_channel_id = chan_4_id;
 
-               match nodes[0].node.send_payment(&route, payment_hash, &None).unwrap_err() {
+               match nodes[0].node.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)).unwrap_err() {
                        PaymentSendFailure::ParameterError(APIError::APIMisuseError { ref err }) => {
                                assert!(regex::Regex::new(r"Payment secret is required for multi-path payments").unwrap().is_match(err))                        },
                        _ => panic!("unexpected error")
@@ -7882,7 +8066,7 @@ pub mod bench {
        use crate::chain::Listen;
        use crate::chain::chainmonitor::{ChainMonitor, Persist};
        use crate::chain::keysinterface::{KeysManager, KeysInterface, InMemorySigner};
-       use crate::ln::channelmanager::{self, BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage};
+       use crate::ln::channelmanager::{self, BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage, PaymentId};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs::{ChannelMessageHandler, Init};
        use crate::routing::gossip::NetworkGraph;
@@ -7900,12 +8084,12 @@ pub mod bench {
        use test::Bencher;
 
        struct NodeHolder<'a, P: Persist<InMemorySigner>> {
-               node: &'a ChannelManager<InMemorySigner,
+               node: &'a ChannelManager<
                        &'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
                                &'a test_utils::TestBroadcaster, &'a test_utils::TestFeeEstimator,
                                &'a test_utils::TestLogger, &'a P>,
                        &'a test_utils::TestBroadcaster, &'a KeysManager,
-                       &'a test_utils::TestFeeEstimator, &'a test_utils::TestLogger>
+                       &'a test_utils::TestFeeEstimator, &'a test_utils::TestLogger>,
        }
 
        #[cfg(test)]
@@ -7988,6 +8172,24 @@ pub mod bench {
                        _ => panic!(),
                }
 
+               let events_a = node_a.get_and_clear_pending_events();
+               assert_eq!(events_a.len(), 1);
+               match events_a[0] {
+                       Event::ChannelReady{ ref counterparty_node_id, .. } => {
+                               assert_eq!(*counterparty_node_id, node_b.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+
+               let events_b = node_b.get_and_clear_pending_events();
+               assert_eq!(events_b.len(), 1);
+               match events_b[0] {
+                       Event::ChannelReady{ ref counterparty_node_id, .. } => {
+                               assert_eq!(*counterparty_node_id, node_a.get_our_node_id());
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+
                let dummy_graph = NetworkGraph::new(genesis_hash, &logger_a);
 
                let mut payment_count: u64 = 0;
@@ -8009,7 +8211,7 @@ pub mod bench {
                                let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
                                let payment_secret = $node_b.create_inbound_payment_for_hash(payment_hash, None, 7200).unwrap();
 
-                               $node_a.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+                               $node_a.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                                let payment_event = SendEvent::from_event($node_a.get_and_clear_pending_msg_events().pop().unwrap());
                                $node_b.handle_update_add_htlc(&$node_a.get_our_node_id(), &payment_event.msgs[0]);
                                $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &payment_event.commitment_msg);
index e8a3e6d26facd524c3ae3036ed87ec54a08b8937..77d0fa4529fb2ea526682079229b81fa9165ddf6 100644 (file)
@@ -157,6 +157,7 @@ mod sealed {
                // Byte 2
                BasicMPP,
        ]);
+       define_context!(OfferContext, []);
        // This isn't a "real" feature context, and is only used in the channel_type field in an
        // `OpenChannel` message.
        define_context!(ChannelTypeContext, [
@@ -366,7 +367,7 @@ mod sealed {
                supports_keysend, requires_keysend);
 
        #[cfg(test)]
-       define_feature!(123456789, UnknownFeature, [NodeContext, ChannelContext, InvoiceContext],
+       define_feature!(123456789, UnknownFeature, [NodeContext, ChannelContext, InvoiceContext, OfferContext],
                "Feature flags for an unknown feature used in testing.", set_unknown_feature_optional,
                set_unknown_feature_required, supports_unknown_test_feature, requires_unknown_test_feature);
 }
@@ -425,6 +426,8 @@ pub type NodeFeatures = Features<sealed::NodeContext>;
 pub type ChannelFeatures = Features<sealed::ChannelContext>;
 /// Features used within an invoice.
 pub type InvoiceFeatures = Features<sealed::InvoiceContext>;
+/// Features used within an offer.
+pub type OfferFeatures = Features<sealed::OfferContext>;
 
 /// Features used within the channel_type field in an OpenChannel message.
 ///
@@ -684,6 +687,15 @@ impl<T: sealed::Wumbo> Features<T> {
        }
 }
 
+#[cfg(test)]
+impl<T: sealed::UnknownFeature> Features<T> {
+       pub(crate) fn unknown() -> Self {
+               let mut features = Self::empty();
+               features.set_unknown_feature_required();
+               features
+       }
+}
+
 macro_rules! impl_feature_len_prefixed_write {
        ($features: ident) => {
                impl Writeable for $features {
@@ -704,21 +716,26 @@ impl_feature_len_prefixed_write!(ChannelFeatures);
 impl_feature_len_prefixed_write!(NodeFeatures);
 impl_feature_len_prefixed_write!(InvoiceFeatures);
 
-// Because ChannelTypeFeatures only appears inside of TLVs, it doesn't have a length prefix when
-// serialized. Thus, we can't use `impl_feature_len_prefixed_write`, above, and have to write our
-// own serialization.
-impl Writeable for ChannelTypeFeatures {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               self.write_be(w)
-       }
-}
-impl Readable for ChannelTypeFeatures {
-       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let v = io_extras::read_to_end(r)?;
-               Ok(Self::from_be_bytes(v))
+// Some features only appear inside of TLVs, so they don't have a length prefix when serialized.
+macro_rules! impl_feature_tlv_write {
+       ($features: ident) => {
+               impl Writeable for $features {
+                       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+                               self.write_be(w)
+                       }
+               }
+               impl Readable for $features {
+                       fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
+                               let v = io_extras::read_to_end(r)?;
+                               Ok(Self::from_be_bytes(v))
+                       }
+               }
        }
 }
 
+impl_feature_tlv_write!(ChannelTypeFeatures);
+impl_feature_tlv_write!(OfferFeatures);
+
 #[cfg(test)]
 mod tests {
        use super::{ChannelFeatures, ChannelTypeFeatures, InitFeatures, InvoiceFeatures, NodeFeatures, sealed};
index 6e6155f682bd6f65afd500e368b60ae1e2b02459..39a457c8b4a975e04a77e8a0e75b6a1f8a131b2e 100644 (file)
@@ -278,7 +278,7 @@ pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub tx_broadcaster: &'c test_utils::TestBroadcaster,
        pub chain_monitor: &'b test_utils::TestChainMonitor<'c>,
        pub keys_manager: &'b test_utils::TestKeysInterface,
-       pub node: &'a ChannelManager<EnforcingSigner, &'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>,
+       pub node: &'a ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>,
        pub network_graph: &'b NetworkGraph<&'c test_utils::TestLogger>,
        pub gossip_sync: P2PGossipSync<&'b NetworkGraph<&'c test_utils::TestLogger>, &'c test_utils::TestChainSource, &'c test_utils::TestLogger>,
        pub node_seed: [u8; 32],
@@ -378,7 +378,7 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
 
                                let mut w = test_utils::TestVecWriter(Vec::new());
                                self.node.write(&mut w).unwrap();
-                               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(w.0), ChannelManagerReadArgs {
+                               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(w.0), ChannelManagerReadArgs {
                                        default_config: *self.node.get_current_default_configuration(),
                                        keys_manager: self.keys_manager,
                                        fee_estimator: &test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) },
@@ -741,6 +741,9 @@ pub fn open_zero_conf_channel<'a, 'b, 'c, 'd>(initiator: &'a Node<'b, 'c, 'd>, r
        assert_eq!(initiator.node.list_usable_channels().len(), initiator_channels + 1);
        assert_eq!(receiver.node.list_usable_channels().len(), receiver_channels + 1);
 
+       expect_channel_ready_event(&initiator, &receiver.node.get_our_node_id());
+       expect_channel_ready_event(&receiver, &initiator.node.get_our_node_id());
+
        (tx, as_channel_ready.channel_id)
 }
 
@@ -794,6 +797,7 @@ pub fn create_chan_between_nodes_with_value_confirm<'a, 'b, 'c, 'd>(node_a: &'a
        create_chan_between_nodes_with_value_confirm_first(node_a, node_b, tx, conf_height);
        confirm_transaction_at(node_a, tx, conf_height);
        connect_blocks(node_a, CHAN_CONFIRM_DEPTH - 1);
+       expect_channel_ready_event(&node_a, &node_b.node.get_our_node_id());
        create_chan_between_nodes_with_value_confirm_second(node_b, node_a)
 }
 
@@ -832,6 +836,7 @@ pub fn create_chan_between_nodes_with_value_b<'a, 'b, 'c>(node_a: &Node<'a, 'b,
 
        *node_a.network_chan_count.borrow_mut() += 1;
 
+       expect_channel_ready_event(&node_b, &node_a.node.get_our_node_id());
        ((*announcement).clone(), (*as_update).clone(), (*bs_update).clone())
 }
 
@@ -870,8 +875,10 @@ pub fn create_unannounced_chan_between_nodes_with_value<'a, 'b, 'c, 'd>(nodes: &
        connect_blocks(&nodes[b], CHAN_CONFIRM_DEPTH - 1);
        let as_channel_ready = get_event_msg!(nodes[a], MessageSendEvent::SendChannelReady, nodes[b].node.get_our_node_id());
        nodes[a].node.handle_channel_ready(&nodes[b].node.get_our_node_id(), &get_event_msg!(nodes[b], MessageSendEvent::SendChannelReady, nodes[a].node.get_our_node_id()));
+       expect_channel_ready_event(&nodes[a], &nodes[b].node.get_our_node_id());
        let as_update = get_event_msg!(nodes[a], MessageSendEvent::SendChannelUpdate, nodes[b].node.get_our_node_id());
        nodes[b].node.handle_channel_ready(&nodes[a].node.get_our_node_id(), &as_channel_ready);
+       expect_channel_ready_event(&nodes[b], &nodes[a].node.get_our_node_id());
        let bs_update = get_event_msg!(nodes[b], MessageSendEvent::SendChannelUpdate, nodes[a].node.get_our_node_id());
 
        nodes[a].node.handle_channel_update(&nodes[b].node.get_our_node_id(), &bs_update);
@@ -1503,6 +1510,19 @@ macro_rules! expect_payment_forwarded {
        }
 }
 
+#[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
+pub fn expect_channel_ready_event<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, expected_counterparty_node_id: &PublicKey) {
+       let events = node.node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 1);
+       match events[0] {
+               crate::util::events::Event::ChannelReady{ ref counterparty_node_id, .. } => {
+                       assert_eq!(*expected_counterparty_node_id, *counterparty_node_id);
+               },
+               _ => panic!("Unexpected event"),
+       }
+}
+
+
 pub struct PaymentFailedConditions<'a> {
        pub(crate) expected_htlc_error_data: Option<(u16, &'a [u8])>,
        pub(crate) expected_blamed_scid: Option<u64>,
@@ -1636,7 +1656,8 @@ pub fn expect_payment_failed_conditions<'a, 'b, 'c, 'd, 'e>(
 }
 
 pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) -> PaymentId {
-       let payment_id = origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       let payment_id = PaymentId(origin_node.keys_manager.backing.get_secure_random_bytes());
+       origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), payment_id).unwrap();
        check_added_monitors!(origin_node, expected_paths.len());
        pass_along_route(origin_node, expected_paths, recv_value, our_payment_hash, our_payment_secret);
        payment_id
@@ -1883,7 +1904,7 @@ pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_rou
        }
 
        let (_, our_payment_hash, our_payment_preimage) = get_payment_preimage_hash!(expected_route.last().unwrap());
-       unwrap_send_err!(origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_preimage)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_preimage), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert!(err.contains("Cannot send value that would put us over the max HTLC value in flight our peer will accept")));
 }
 
@@ -1904,6 +1925,22 @@ pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
 }
 
 pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash) {
+       let expected_payment_id = pass_failed_payment_back_no_abandon(origin_node, expected_paths_slice, skip_last, our_payment_hash);
+       if !skip_last {
+               origin_node.node.abandon_payment(expected_payment_id.unwrap());
+               let events = origin_node.node.get_and_clear_pending_events();
+               assert_eq!(events.len(), 1);
+               match events[0] {
+                       Event::PaymentFailed { ref payment_hash, ref payment_id } => {
+                               assert_eq!(*payment_hash, our_payment_hash, "unexpected second payment_hash");
+                               assert_eq!(*payment_id, expected_payment_id.unwrap());
+                       }
+                       _ => panic!("Unexpected second event"),
+               }
+       }
+}
+
+pub fn pass_failed_payment_back_no_abandon<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash) -> Option<PaymentId> {
        let mut expected_paths: Vec<_> = expected_paths_slice.iter().collect();
        check_added_monitors!(expected_paths[0].last().unwrap(), expected_paths.len());
 
@@ -1927,6 +1964,8 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
        per_path_msgs.sort_unstable_by(|(_, node_id_a), (_, node_id_b)| node_id_a.cmp(node_id_b));
        expected_paths.sort_unstable_by(|path_a, path_b| path_a[path_a.len() - 2].node.get_our_node_id().cmp(&path_b[path_b.len() - 2].node.get_our_node_id()));
 
+       let mut expected_payment_id = None;
+
        for (i, (expected_route, (path_msgs, next_hop))) in expected_paths.iter().zip(per_path_msgs.drain(..)).enumerate() {
                let mut next_msgs = Some(path_msgs);
                let mut expected_next_node = next_hop;
@@ -1975,7 +2014,7 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
                        commitment_signed_dance!(origin_node, prev_node, next_msgs.as_ref().unwrap().1, false);
                        let events = origin_node.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
-                       let expected_payment_id = match events[0] {
+                       expected_payment_id = Some(match events[0] {
                                Event::PaymentPathFailed { payment_hash, payment_failed_permanently, all_paths_failed, ref path, ref payment_id, .. } => {
                                        assert_eq!(payment_hash, our_payment_hash);
                                        assert!(payment_failed_permanently);
@@ -1986,19 +2025,7 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
                                        payment_id.unwrap()
                                },
                                _ => panic!("Unexpected event"),
-                       };
-                       if i == expected_paths.len() - 1 {
-                               origin_node.node.abandon_payment(expected_payment_id);
-                               let events = origin_node.node.get_and_clear_pending_events();
-                               assert_eq!(events.len(), 1);
-                               match events[0] {
-                                       Event::PaymentFailed { ref payment_hash, ref payment_id } => {
-                                               assert_eq!(*payment_hash, our_payment_hash, "unexpected second payment_hash");
-                                               assert_eq!(*payment_id, expected_payment_id);
-                                       }
-                                       _ => panic!("Unexpected second event"),
-                               }
-                       }
+                       });
                }
        }
 
@@ -2007,6 +2034,8 @@ pub fn pass_failed_payment_back<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expe
        assert!(expected_paths[0].last().unwrap().node.get_and_clear_pending_events().is_empty());
        assert!(expected_paths[0].last().unwrap().node.get_and_clear_pending_msg_events().is_empty());
        check_added_monitors!(expected_paths[0].last().unwrap(), 0);
+
+       expected_payment_id
 }
 
 pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], our_payment_hash: PaymentHash)  {
@@ -2071,7 +2100,7 @@ pub fn test_default_channel_config() -> UserConfig {
        default_config
 }
 
-pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>>, node_config: &[Option<UserConfig>]) -> Vec<ChannelManager<EnforcingSigner, &'a TestChainMonitor<'b>, &'b test_utils::TestBroadcaster, &'a test_utils::TestKeysInterface, &'b test_utils::TestFeeEstimator, &'b test_utils::TestLogger>> {
+pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>>, node_config: &[Option<UserConfig>]) -> Vec<ChannelManager<&'a TestChainMonitor<'b>, &'b test_utils::TestBroadcaster, &'a test_utils::TestKeysInterface, &'b test_utils::TestFeeEstimator, &'b test_utils::TestLogger>> {
        let mut chanmgrs = Vec::new();
        for i in 0..node_count {
                let network = Network::Testnet;
@@ -2087,7 +2116,7 @@ pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>
        chanmgrs
 }
 
-pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeCfg<'c>>, chan_mgrs: &'a Vec<ChannelManager<EnforcingSigner, &'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>>) -> Vec<Node<'a, 'b, 'c>> {
+pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeCfg<'c>>, chan_mgrs: &'a Vec<ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>>) -> Vec<Node<'a, 'b, 'c>> {
        let mut nodes = Vec::new();
        let chan_count = Rc::new(RefCell::new(0));
        let payment_count = Rc::new(RefCell::new(0));
index 5d7049b79bbf475137633b010ff7a940ef2130b5..b95356a3dd7aa8b7375118612fe2faa3c2a8b2e7 100644 (file)
@@ -20,7 +20,7 @@ use crate::chain::transaction::OutPoint;
 use crate::chain::keysinterface::{BaseSign, KeysInterface};
 use crate::ln::{PaymentPreimage, PaymentSecret, PaymentHash};
 use crate::ln::channel::{commitment_tx_base_weight, COMMITMENT_TX_WEIGHT_PER_HTLC, CONCURRENT_INBOUND_HTLC_FEE_BUFFER, FEE_SPIKE_BUFFER_FEE_INCREASE_MULTIPLE, MIN_AFFORDABLE_HTLC_COUNT};
-use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, PaymentId, RAACommitmentOrder, PaymentSendFailure, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA, PAYMENT_EXPIRY_BLOCKS};
+use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, PaymentId, RAACommitmentOrder, PaymentSendFailure, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA};
 use crate::ln::channel::{Channel, ChannelError};
 use crate::ln::{chan_utils, onion_utils};
 use crate::ln::chan_utils::{OFFERED_HTLC_SCRIPT_WEIGHT, htlc_success_tx_weight, htlc_timeout_tx_weight, HTLCOutputInCommitment};
@@ -253,7 +253,7 @@ fn test_async_inbound_update_fee() {
 
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 40000);
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -352,7 +352,7 @@ fn test_update_fee_unordered_raa() {
 
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 40000);
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -558,6 +558,7 @@ fn do_test_sanity_on_in_flight_opens(steps: u8) {
        confirm_transaction_at(&nodes[0], &tx, 2);
        connect_blocks(&nodes[0], CHAN_CONFIRM_DEPTH);
        create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
+       expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
 }
 
 #[test]
@@ -781,7 +782,7 @@ fn test_update_fee_with_fundee_update_add_htlc() {
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 800000);
 
        // nothing happens since node[1] is in AwaitingRemoteRevoke
-       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        {
                let mut added_monitors = nodes[0].chain_monitor.added_monitors.lock().unwrap();
                assert_eq!(added_monitors.len(), 0);
@@ -1093,7 +1094,7 @@ fn holding_cell_htlc_counting() {
        let mut payments = Vec::new();
        for _ in 0..crate::ln::channel::OUR_MAX_HTLCS {
                let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                payments.push((payment_preimage, payment_hash));
        }
        check_added_monitors!(nodes[1], 1);
@@ -1108,7 +1109,7 @@ fn holding_cell_htlc_counting() {
        // another HTLC.
        let (route, payment_hash_1, _, payment_secret_1) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
        {
-               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
                nodes[1].logger.assert_log_contains("lightning::ln::channelmanager".to_string(), "Cannot push more than their max accepted HTLCs".to_string(), 1);
@@ -1117,7 +1118,7 @@ fn holding_cell_htlc_counting() {
        // This should also be true if we try to forward a payment.
        let (route, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 100000);
        {
-               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
 
@@ -1330,7 +1331,7 @@ fn test_basic_channel_reserve() {
        let commit_tx_fee = 2 * commit_tx_fee_msat(get_feerate!(nodes[0], chan.2), 1 + 1, get_opt_anchors!(nodes[0], chan.2));
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send + 1);
-       let err = nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).err().unwrap();
+       let err = nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).err().unwrap();
        match err {
                PaymentSendFailure::AllFailedRetrySafe(ref fails) => {
                        match &fails[0] {
@@ -1506,7 +1507,7 @@ fn test_chan_reserve_violation_outbound_htlc_inbound_chan() {
 
        // However one more HTLC should be significantly over the reserve amount and fail.
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 1_000_000);
-       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send value that would put counterparty balance under holder-announced channel reserve value"));
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Cannot send value that would put counterparty balance under holder-announced channel reserve value".to_string(), 1);
@@ -1597,7 +1598,7 @@ fn test_chan_reserve_dust_inbound_htlcs_outbound_chan() {
 
        // One more than the dust amt should fail, however.
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], dust_amt + 1);
-       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send value that would put counterparty balance under holder-announced channel reserve value"));
 }
 
@@ -1691,7 +1692,7 @@ fn test_chan_reserve_violation_inbound_htlc_inbound_chan() {
        // Add a pending HTLC.
        let (route_1, our_payment_hash_1, _, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[2], amt_msat_1);
        let payment_event_1 = {
-               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1801,7 +1802,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
                route.paths[0].last_mut().unwrap().fee_msat += 1;
                assert!(route.paths[0].iter().rev().skip(1).all(|h| h.fee_msat == feemsat));
 
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put us over the max HTLC value in flight our peer will accept \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
                nodes[0].logger.assert_log_contains("lightning::ln::channelmanager".to_string(), "Cannot send value that would put us over the max HTLC value in flight our peer will accept".to_string(), 1);
@@ -1857,7 +1858,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
 
        let (route_1, our_payment_hash_1, our_payment_preimage_1, our_payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_1);
        let payment_event_1 = {
-               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route_1, our_payment_hash_1, &Some(our_payment_secret_1), PaymentId(our_payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1870,7 +1871,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
        let recv_value_2 = stat01.value_to_self_msat - amt_msat_1 - stat01.channel_reserve_msat - total_fee_msat - commit_tx_fee_2_htlcs;
        {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_2 + 1);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put our balance under counterparty-announced channel reserve value \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        }
@@ -1888,7 +1889,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
        // now see if they go through on both sides
        let (route_21, our_payment_hash_21, our_payment_preimage_21, our_payment_secret_21) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_21);
        // but this will stuck in the holding cell
-       nodes[0].node.send_payment(&route_21, our_payment_hash_21, &Some(our_payment_secret_21)).unwrap();
+       nodes[0].node.send_payment(&route_21, our_payment_hash_21, &Some(our_payment_secret_21), PaymentId(our_payment_hash_21.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 0);
@@ -1896,7 +1897,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
        // test with outbound holding cell amount > 0
        {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_22+1);
-               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+               unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot send value that would put our balance under counterparty-announced channel reserve value \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
                nodes[0].logger.assert_log_contains("lightning::ln::channelmanager".to_string(), "Cannot send value that would put our balance under counterparty-announced channel reserve value".to_string(), 2);
@@ -1904,7 +1905,7 @@ fn test_channel_reserve_holding_cell_htlcs() {
 
        let (route_22, our_payment_hash_22, our_payment_preimage_22, our_payment_secret_22) = get_route_and_payment_hash!(nodes[0], nodes[2], recv_value_22);
        // this will also stuck in the holding cell
-       nodes[0].node.send_payment(&route_22, our_payment_hash_22, &Some(our_payment_secret_22)).unwrap();
+       nodes[0].node.send_payment(&route_22, our_payment_hash_22, &Some(our_payment_secret_22), PaymentId(our_payment_hash_22.0)).unwrap();
        check_added_monitors!(nodes[0], 0);
        assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -2040,7 +2041,7 @@ fn channel_reserve_in_flight_removes() {
        // Start routing the third HTLC (this is just used to get everyone in the right state).
        let (route, payment_hash_3, payment_preimage_3, payment_secret_3) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
        let send_1 = {
-               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3), PaymentId(payment_hash_3.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -2114,7 +2115,7 @@ fn channel_reserve_in_flight_removes() {
        // to A to ensure that A doesn't count the almost-removed HTLC in update_add processing.
        let (route, payment_hash_4, payment_preimage_4, payment_secret_4) = get_route_and_payment_hash!(nodes[1], nodes[0], 10000);
        let send_2 = {
-               nodes[1].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4)).unwrap();
+               nodes[1].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4), PaymentId(payment_hash_4.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
                let mut events = nodes[1].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -2951,26 +2952,8 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
        mine_transaction(&nodes[1], &timeout_tx);
        check_added_monitors!(nodes[1], 1);
        check_closed_broadcast!(nodes[1], true);
-       {
-               // B will rebroadcast a fee-bumped timeout transaction here.
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-               assert_eq!(node_txn.len(), 1);
-               check_spends!(node_txn[0], commitment_tx[0]);
-       }
 
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
-       {
-               // B may rebroadcast its own holder commitment transaction here, as a safeguard against
-               // some incredibly unlikely partial-eclipse-attack scenarios. That said, because the
-               // original commitment_tx[0] (also spending chan_2.3) has reached ANTI_REORG_DELAY B really
-               // shouldn't broadcast anything here, and in some connect style scenarios we do not.
-               let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
-               if node_txn.len() == 1 {
-                       check_spends!(node_txn[0], chan_2.3);
-               } else {
-                       assert_eq!(node_txn.len(), 0);
-               }
-       }
 
        expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], vec![HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_2.2 }]);
        check_added_monitors!(nodes[1], 1);
@@ -3154,7 +3137,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        // Add a fourth HTLC, this one will get sequestered away in nodes[1]'s holding cell waiting
        // on nodes[2]'s RAA.
        let (route, fourth_payment_hash, _, fourth_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 1000000);
-       nodes[1].node.send_payment(&route, fourth_payment_hash, &Some(fourth_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&route, fourth_payment_hash, &Some(fourth_payment_secret), PaymentId(fourth_payment_hash.0)).unwrap();
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
        check_added_monitors!(nodes[1], 0);
@@ -3184,10 +3167,9 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        check_added_monitors!(nodes[1], 1);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
-       assert!(ANTI_REORG_DELAY > PAYMENT_EXPIRY_BLOCKS); // We assume payments will also expire
 
        let events = nodes[1].node.get_and_clear_pending_events();
-       assert_eq!(events.len(), if deliver_bs_raa { 2 + (nodes.len() - 1) } else { 4 + nodes.len() });
+       assert_eq!(events.len(), if deliver_bs_raa { 2 + nodes.len() - 1 } else { 3 + nodes.len() });
        match events[0] {
                Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => { },
                _ => panic!("Unexepected event"),
@@ -3200,15 +3182,18 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        }
        if !deliver_bs_raa {
                match events[2] {
+                       Event::PendingHTLCsForwardable { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+               nodes[1].node.abandon_payment(PaymentId(fourth_payment_hash.0));
+               let payment_failed_events = nodes[1].node.get_and_clear_pending_events();
+               assert_eq!(payment_failed_events.len(), 1);
+               match payment_failed_events[0] {
                        Event::PaymentFailed { ref payment_hash, .. } => {
                                assert_eq!(*payment_hash, fourth_payment_hash);
                        },
                        _ => panic!("Unexpected event"),
                }
-               match events[3] {
-                       Event::PendingHTLCsForwardable { .. } => { },
-                       _ => panic!("Unexpected event"),
-               };
        }
        nodes[1].node.process_pending_htlc_forwards();
        check_added_monitors!(nodes[1], 1);
@@ -3315,7 +3300,7 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        // Alice -> Bob: Route a payment but without Bob sending revoke_and_ack.
        {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 50_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let payment_event = {
@@ -3330,7 +3315,7 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        // Alice -> Bob: Route another payment but now Alice waits for Bob's earlier revoke_and_ack.
        let (route, failed_payment_hash, _, failed_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 50_000);
        {
-               nodes[0].node.send_payment(&route, failed_payment_hash, &Some(failed_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, failed_payment_hash, &Some(failed_payment_secret), PaymentId(failed_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 0);
 
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -3423,7 +3408,7 @@ fn test_force_close_fail_back() {
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1000000);
 
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -3640,7 +3625,7 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let (route, payment_hash_1, payment_preimage_1, payment_secret_1) = get_route_and_payment_hash!(nodes[0], nodes[1], 1_000_000);
 
        let payment_event = {
-               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -3711,11 +3696,23 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        }
 
        let events_1 = nodes[1].node.get_and_clear_pending_events();
-       assert_eq!(events_1.len(), 1);
-       match events_1[0] {
-               Event::PendingHTLCsForwardable { .. } => { },
-               _ => panic!("Unexpected event"),
-       };
+       if messages_delivered == 0 {
+               assert_eq!(events_1.len(), 2);
+               match events_1[0] {
+                       Event::ChannelReady { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+               match events_1[1] {
+                       Event::PendingHTLCsForwardable { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+       } else {
+               assert_eq!(events_1.len(), 1);
+               match events_1[0] {
+                       Event::PendingHTLCsForwardable { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+       }
 
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
@@ -3863,7 +3860,7 @@ fn test_funding_peer_disconnect() {
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
@@ -3952,6 +3949,8 @@ fn test_funding_peer_disconnect() {
                },
                _ => panic!("Unexpected event {:?}", events_6[0]),
        };
+       expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
+       expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 
        // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
        // broadcast the channel announcement globally, as well as re-send its (now-public)
@@ -4009,7 +4008,7 @@ fn test_funding_peer_disconnect() {
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: UserConfig::default(),
                        keys_manager,
                        fee_estimator: node_cfgs[0].fee_estimator,
@@ -4070,7 +4069,7 @@ fn test_drop_messages_peer_disconnect_dual_htlc() {
 
        // Now try to send a second payment which will fail to send
        let (route, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
@@ -4222,7 +4221,8 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) {
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
                let payment_id = PaymentId([42; 32]);
-               nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200000, cur_height, payment_id, &None).unwrap();
+               let session_privs = nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &route).unwrap();
+               nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -4287,7 +4287,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
        // Route a first payment to get the 1 -> 2 channel in awaiting_raa...
        let (route, first_payment_hash, _, first_payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[2], 100000);
        {
-               nodes[1].node.send_payment(&route, first_payment_hash, &Some(first_payment_secret)).unwrap();
+               nodes[1].node.send_payment(&route, first_payment_hash, &Some(first_payment_secret), PaymentId(first_payment_hash.0)).unwrap();
        }
        assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
        check_added_monitors!(nodes[1], 1);
@@ -4295,7 +4295,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
        // Now attempt to route a second payment, which should be placed in the holding cell
        let sending_node = if forwarded_htlc { &nodes[0] } else { &nodes[1] };
        let (route, second_payment_hash, _, second_payment_secret) = get_route_and_payment_hash!(sending_node, nodes[2], 100000);
-       sending_node.node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)).unwrap();
+       sending_node.node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), PaymentId(second_payment_hash.0)).unwrap();
        if forwarded_htlc {
                check_added_monitors!(nodes[0], 1);
                let payment_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
@@ -4324,14 +4324,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
                }
                expect_payment_failed_with_update!(nodes[0], second_payment_hash, false, chan_2.0.contents.short_channel_id, false);
        } else {
-               let events = nodes[1].node.get_and_clear_pending_events();
-               assert_eq!(events.len(), 2);
-               if let Event::PaymentPathFailed { ref payment_hash, .. } = events[0] {
-                       assert_eq!(*payment_hash, second_payment_hash);
-               } else { panic!("Unexpected event"); }
-               if let Event::PaymentFailed { ref payment_hash, .. } = events[1] {
-                       assert_eq!(*payment_hash, second_payment_hash);
-               } else { panic!("Unexpected event"); }
+               expect_payment_failed!(nodes[1], second_payment_hash, false);
        }
 }
 
@@ -4350,7 +4343,7 @@ fn test_no_txn_manager_serialize_deserialize() {
        let fee_estimator: test_utils::TestFeeEstimator;
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
@@ -4378,7 +4371,7 @@ fn test_no_txn_manager_serialize_deserialize() {
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: config,
                        keys_manager,
                        fee_estimator: &fee_estimator,
@@ -4428,7 +4421,7 @@ fn test_manager_serialize_deserialize_events() {
        let persister: test_utils::TestPersister;
        let logger: test_utils::TestLogger;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        // Start creating a channel, but stop right before broadcasting the funding transaction
@@ -4489,7 +4482,7 @@ fn test_manager_serialize_deserialize_events() {
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: config,
                        keys_manager,
                        fee_estimator: &fee_estimator,
@@ -4548,7 +4541,7 @@ fn test_simple_manager_serialize_deserialize() {
        let fee_estimator: test_utils::TestFeeEstimator;
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 
@@ -4576,7 +4569,7 @@ fn test_simple_manager_serialize_deserialize() {
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: UserConfig::default(),
                        keys_manager,
                        fee_estimator: &fee_estimator,
@@ -4610,7 +4603,7 @@ fn test_manager_serialize_deserialize_inconsistent_monitor() {
        let fee_estimator: test_utils::TestFeeEstimator;
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
        let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
        let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
@@ -4668,7 +4661,7 @@ fn test_manager_serialize_deserialize_inconsistent_monitor() {
 
        let mut nodes_0_read = &nodes_0_serialized[..];
        if let Err(msgs::DecodeError::InvalidValue) =
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                default_config: UserConfig::default(),
                keys_manager,
                fee_estimator: &fee_estimator,
@@ -4682,7 +4675,7 @@ fn test_manager_serialize_deserialize_inconsistent_monitor() {
 
        let mut nodes_0_read = &nodes_0_serialized[..];
        let (_, nodes_0_deserialized_tmp) =
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                default_config: UserConfig::default(),
                keys_manager,
                fee_estimator: &fee_estimator,
@@ -5976,7 +5969,7 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], if use_dust { 50000 } else { 3000000 });
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let _as_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6046,14 +6039,7 @@ fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no
                check_added_monitors!(nodes[0], 1);
                check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        } else {
-               let events = nodes[0].node.get_and_clear_pending_events();
-               assert_eq!(events.len(), 2);
-               if let Event::PaymentPathFailed { ref payment_hash, .. } = events[0] {
-                       assert_eq!(*payment_hash, our_payment_hash);
-               } else { panic!("Unexpected event"); }
-               if let Event::PaymentFailed { ref payment_hash, .. } = events[1] {
-                       assert_eq!(*payment_hash, our_payment_hash);
-               } else { panic!("Unexpected event"); }
+               expect_payment_failed!(nodes[0], our_payment_hash, true);
        }
 }
 
@@ -6222,7 +6208,7 @@ fn test_fail_holding_cell_htlc_upon_free() {
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send);
 
        // Send a payment which passes reserve checks but gets stuck in the holding cell.
-       let our_payment_id = nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, max_can_send);
 
@@ -6248,7 +6234,7 @@ fn test_fail_holding_cell_htlc_upon_free() {
        assert_eq!(events.len(), 1);
        match &events[0] {
                &Event::PaymentPathFailed { ref payment_id, ref payment_hash, ref payment_failed_permanently, ref network_update, ref all_paths_failed, ref short_channel_id, .. } => {
-                       assert_eq!(our_payment_id, *payment_id.as_ref().unwrap());
+                       assert_eq!(PaymentId(our_payment_hash.0), *payment_id.as_ref().unwrap());
                        assert_eq!(our_payment_hash.clone(), *payment_hash);
                        assert_eq!(*payment_failed_permanently, false);
                        assert_eq!(*all_paths_failed, true);
@@ -6302,10 +6288,11 @@ fn test_free_and_fail_holding_cell_htlcs() {
        let (route_2, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], amt_2);
 
        // Send 2 payments which pass reserve checks but get stuck in the holding cell.
-       nodes[0].node.send_payment(&route_1, payment_hash_1, &Some(payment_secret_1)).unwrap();
+       nodes[0].node.send_payment(&route_1, payment_hash_1, &Some(payment_secret_1), PaymentId(payment_hash_1.0)).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, amt_1);
-       let payment_id_2 = nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       let payment_id_2 = PaymentId(nodes[0].keys_manager.get_secure_random_bytes());
+       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), payment_id_2).unwrap();
        chan_stat = get_channel_value_stat!(nodes[0], chan.2);
        assert_eq!(chan_stat.holding_cell_outbound_amount_msat, amt_1 + amt_2);
 
@@ -6427,7 +6414,7 @@ fn test_fail_holding_cell_htlc_upon_free_multihop() {
        let max_can_send = 5000000 - channel_reserve - 2*commit_tx_fee_msat(feerate, 1 + 1, opt_anchors) - total_routing_fee_msat;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], max_can_send);
        let payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -6529,7 +6516,7 @@ fn test_update_add_htlc_bolt2_sender_value_below_minimum_msat() {
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
        route.paths[0][0].fee_msat = 100;
 
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot send less than their minimum HTLC value \(\d+\)").unwrap().is_match(err)));
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        nodes[0].logger.assert_log_contains("lightning::ln::channelmanager".to_string(), "Cannot send less than their minimum HTLC value".to_string(), 1);
@@ -6546,7 +6533,7 @@ fn test_update_add_htlc_bolt2_sender_zero_value_msat() {
 
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
        route.paths[0][0].fee_msat = 0;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send 0-msat HTLC"));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6563,7 +6550,7 @@ fn test_update_add_htlc_bolt2_receiver_zero_value_msat() {
        let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = 0;
@@ -6589,7 +6576,7 @@ fn test_update_add_htlc_bolt2_sender_cltv_expiry_too_high() {
                .with_features(channelmanager::provided_invoice_features());
        let (mut route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], payment_params, 100000000, 0);
        route.paths[0].last_mut().unwrap().cltv_expiry_delta = 500000001;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::RouteError { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::RouteError { ref err },
                assert_eq!(err, &"Channel CLTV overflowed?"));
 }
 
@@ -6608,7 +6595,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
        for i in 0..max_accepted_htlcs {
                let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
                let payment_event = {
-                       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+                       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                        check_added_monitors!(nodes[0], 1);
 
                        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -6628,7 +6615,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
                expect_payment_received!(nodes[1], our_payment_hash, our_payment_secret, 100000);
        }
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100000);
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6652,7 +6639,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_value_in_flight() {
        // Manually create a route over our max in flight (which our router normally automatically
        // limits us to.
        route.paths[0][0].fee_msat =  max_in_flight + 1;
-       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
+       unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot send value that would put us over the max HTLC value in flight our peer will accept \(\d+\)").unwrap().is_match(err)));
 
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -6678,7 +6665,7 @@ fn test_update_add_htlc_bolt2_receiver_check_amount_received_more_than_min() {
        }
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], htlc_minimum_msat);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = htlc_minimum_msat-1;
@@ -6708,7 +6695,7 @@ fn test_update_add_htlc_bolt2_receiver_sender_can_afford_amount_sent() {
 
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee_outbound;
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], max_can_send);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 
@@ -6775,7 +6762,7 @@ fn test_update_add_htlc_bolt2_receiver_check_max_in_flight_msat() {
        let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].amount_msat = get_channel_value_stat!(nodes[1], chan.2).counterparty_max_htlc_value_in_flight_msat + 1;
@@ -6798,7 +6785,7 @@ fn test_update_add_htlc_bolt2_receiver_check_cltv_expiry() {
 
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        updates.update_add_htlcs[0].cltv_expiry = 500000000;
@@ -6823,7 +6810,7 @@ fn test_update_add_htlc_bolt2_receiver_check_repeated_id_ignore() {
 
        create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -6868,7 +6855,7 @@ fn test_update_fulfill_htlc_bolt2_update_fulfill_htlc_before_commitment() {
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
 
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6900,7 +6887,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_htlc_before_commitment() {
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -6931,7 +6918,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_malformed_htlc_before_commitment()
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
@@ -7048,7 +7035,7 @@ fn test_update_fulfill_htlc_bolt2_missing_badonion_bit_for_malformed_htlc_messag
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -7099,7 +7086,7 @@ fn test_update_fulfill_htlc_bolt2_after_malformed_htlc_message_must_forward_upda
 
        //First hop
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -7173,7 +7160,7 @@ fn test_channel_failed_after_message_with_badonion_node_perm_bits_set() {
 
        // First hop
        let mut payment_event = {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                SendEvent::from_node(&nodes[0])
        };
@@ -7528,7 +7515,7 @@ fn do_test_data_loss_protect(reconnect_panicing: bool) {
        node_state_0 = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(OutPoint { txid: chan.3.txid(), index: 0 }, &mut chain_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(previous_node_state), ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(previous_node_state), ChannelManagerReadArgs {
                        keys_manager: keys_manager,
                        fee_estimator: &fee_estimator,
                        chain_monitor: &monitor,
@@ -7654,7 +7641,7 @@ fn test_check_htlc_underpaying() {
        let route = get_route(&nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(), None, 10_000, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
        let (_, our_payment_hash, _) = get_payment_preimage_hash!(nodes[0]);
        let our_payment_secret = nodes[1].node.create_inbound_payment_for_hash(our_payment_hash, Some(100_000), 7200).unwrap();
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -8001,22 +7988,6 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        connect_block(&nodes[0], &Block { header: header_130, txdata: penalty_txn });
        let header_131 = BlockHeader { version: 0x20000000, prev_blockhash: header_130.block_hash(), merkle_root: TxMerkleNode::all_zeros(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[0], &Block { header: header_131, txdata: Vec::new() });
-       {
-               let mut node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
-               assert_eq!(node_txn.len(), 2); // 2 bumped penalty txn on revoked commitment tx
-
-               check_spends!(node_txn[0], revoked_local_txn[0]);
-               check_spends!(node_txn[1], revoked_local_txn[0]);
-               // Note that these are both bogus - they spend outputs already claimed in block 129:
-               if node_txn[0].input[0].previous_output == revoked_htlc_txn[0].input[0].previous_output  {
-                       assert_eq!(node_txn[1].input[0].previous_output, revoked_htlc_txn[2].input[0].previous_output);
-               } else {
-                       assert_eq!(node_txn[0].input[0].previous_output, revoked_htlc_txn[2].input[0].previous_output);
-                       assert_eq!(node_txn[1].input[0].previous_output, revoked_htlc_txn[0].input[0].previous_output);
-               }
-
-               node_txn.clear();
-       };
 
        // Few more blocks to confirm penalty txn
        connect_blocks(&nodes[0], 4);
@@ -8291,7 +8262,7 @@ fn test_pending_claimed_htlc_no_balance_underflow() {
        route.payment_params = None; // This is all wrong, but unnecessary
        route.paths[0][0].pubkey = nodes[0].node.get_our_node_id();
        let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
-       nodes[1].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[1].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 
        assert_eq!(nodes[1].node.list_channels()[0].balance_msat, 1_000_000);
 }
@@ -8674,7 +8645,7 @@ fn test_preimage_storage() {
        {
                let (payment_hash, payment_secret) = nodes[1].node.create_inbound_payment(Some(100_000), 7200).unwrap();
                let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                let mut payment_event = SendEvent::from_event(events.pop().unwrap());
@@ -8744,7 +8715,7 @@ fn test_secret_timeout() {
 
        {
                let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-               nodes[0].node.send_payment(&route, payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, payment_hash, &Some(our_payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                let mut payment_event = SendEvent::from_event(events.pop().unwrap());
@@ -8814,17 +8785,17 @@ fn test_bad_secret_hash() {
        let expected_error_data = [0, 0, 0, 0, 0, 1, 0x86, 0xa0, 0, 0, 0, CHAN_CONFIRM_DEPTH as u8];
 
        // Send a payment with the right payment hash but the wrong payment secret
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(random_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(random_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(our_payment_hash);
        expect_payment_failed!(nodes[0], our_payment_hash, true, expected_error_code, expected_error_data);
 
        // Send a payment with a random payment hash, but the right payment secret
-       nodes[0].node.send_payment(&route, random_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, random_payment_hash, &Some(our_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(random_payment_hash);
        expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data);
 
        // Send a payment with a random payment hash and random payment secret
-       nodes[0].node.send_payment(&route, random_payment_hash, &Some(random_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, random_payment_hash, &Some(random_payment_secret), PaymentId(random_payment_hash.0)).unwrap();
        handle_unknown_invalid_payment_data!(random_payment_hash);
        expect_payment_failed!(nodes[0], random_payment_hash, true, expected_error_code, expected_error_data);
 }
@@ -8969,7 +8940,7 @@ fn test_concurrent_monitor_claim() {
        // Route another payment to generate another update with still previous HTLC pending
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 3000000);
        {
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        }
        check_added_monitors!(nodes[1], 1);
 
@@ -9429,6 +9400,7 @@ fn test_duplicate_chan_id() {
        let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
        let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
        update_nodes_with_chan_announce(&nodes, 0, 1, &announcement, &as_update, &bs_update);
+
        send_payment(&nodes[0], &[&nodes[1]], 8000000);
 }
 
@@ -9679,14 +9651,14 @@ fn test_forwardable_regen() {
        let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_1_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
        let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
        let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 
        // First send a payment to nodes[1]
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9699,7 +9671,7 @@ fn test_forwardable_regen() {
 
        // Next send a payment which is forwarded by nodes[1]
        let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9741,7 +9713,7 @@ fn test_forwardable_regen() {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
                channel_monitors.insert(chan_1_monitor.get_funding_txo().0, &mut chan_1_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
                        default_config: UserConfig::default(),
                        keys_manager,
                        fee_estimator: node_cfgs[1].fee_estimator,
@@ -9799,7 +9771,7 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(&nodes[1]);
 
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -9811,7 +9783,8 @@ fn do_test_dup_htlc_second_rejected(test_for_second_fail_panic: bool) {
        expect_payment_received!(nodes[1], our_payment_hash, our_payment_secret, 10_000);
 
        {
-               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+               // Note that we use a different PaymentId here to allow us to duplicativly pay
+               nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_secret.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -9911,8 +9884,16 @@ fn test_inconsistent_mpp_params() {
 
        let cur_height = nodes[0].best_block_info().1;
        let payment_id = PaymentId([42; 32]);
+
+       let session_privs = {
+               // We create a fake route here so that we start with three pending HTLCs, which we'll
+               // ultimately have, just not right away.
+               let mut dup_route = route.clone();
+               dup_route.paths.push(route.paths[1].clone());
+               nodes[0].node.test_add_new_pending_payment(our_payment_hash, Some(our_payment_secret), payment_id, &dup_route).unwrap()
+       };
        {
-               nodes[0].node.send_payment_along_path(&route.paths[0], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None).unwrap();
+               nodes[0].node.send_payment_along_path(&route.paths[0], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9922,7 +9903,7 @@ fn test_inconsistent_mpp_params() {
        assert!(nodes[3].node.get_and_clear_pending_events().is_empty());
 
        {
-               nodes[0].node.send_payment_along_path(&route.paths[1], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 14_000_000, cur_height, payment_id, &None).unwrap();
+               nodes[0].node.send_payment_along_path(&route.paths[1], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 14_000_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
                check_added_monitors!(nodes[0], 1);
 
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9967,7 +9948,7 @@ fn test_inconsistent_mpp_params() {
 
        expect_payment_failed_conditions(&nodes[0], our_payment_hash, true, PaymentFailedConditions::new().mpp_parts_remain());
 
-       nodes[0].node.send_payment_along_path(&route.paths[1], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None).unwrap();
+       nodes[0].node.send_payment_along_path(&route.paths[1], &payment_params_opt, &our_payment_hash, &Some(our_payment_secret), 15_000_000, cur_height, payment_id, &None, session_privs[2]).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -9998,7 +9979,7 @@ fn test_keysend_payments_to_public_node() {
        let route = find_route(&payer_pubkey, &route_params, &network_graph, None, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
 
        let test_preimage = PaymentPreimage([42; 32]);
-       let (payment_hash, _) = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage)).unwrap();
+       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage), PaymentId(test_preimage.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -10036,7 +10017,7 @@ fn test_keysend_payments_to_private_node() {
        ).unwrap();
 
        let test_preimage = PaymentPreimage([42; 32]);
-       let (payment_hash, _) = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage)).unwrap();
+       let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(test_preimage), PaymentId(test_preimage.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -10087,7 +10068,7 @@ fn test_double_partial_claim() {
        pass_failed_payment_back(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
 
        // nodes[1] now retries one of the two paths...
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -10121,7 +10102,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_3_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 
        let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 
@@ -10139,7 +10120,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
                        core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
        });
 
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2);
 
        // Send the payment through to nodes[3] *without* clearing the PaymentReceived event
@@ -10206,7 +10187,7 @@ fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
                for monitor in monitors.iter_mut() {
                        channel_monitors.insert(monitor.get_funding_txo().0, monitor);
                }
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut &original_manager.0[..], ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut &original_manager.0[..], ChannelManagerReadArgs {
                        default_config: config,
                        keys_manager,
                        fee_estimator: node_cfgs[3].fee_estimator,
@@ -10378,7 +10359,7 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                        // Note, we need sent payment to be above outbound dust threshold on counterparty_tx of 2132 sats
                        for i in 0..dust_outbound_htlc_on_holder_tx {
                                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], dust_outbound_htlc_on_holder_tx_msat);
-                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) { panic!("Unexpected event at dust HTLC {}", i); }
+                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at dust HTLC {}", i); }
                        }
                } else {
                        // Inbound dust threshold: 2324 sats (`dust_buffer_feerate` * HTLC_SUCCESS_TX_WEIGHT / 1000 + holder's `dust_limit_satoshis`)
@@ -10394,7 +10375,7 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                        // Outbound dust balance: 5000 sats
                        for i in 0..dust_htlc_on_counterparty_tx {
                                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], dust_htlc_on_counterparty_tx_msat);
-                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) { panic!("Unexpected event at dust HTLC {}", i); }
+                               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at dust HTLC {}", i); }
                        }
                } else {
                        // Inbound dust threshold: 2031 sats (`dust_buffer_feerate` * HTLC_TIMEOUT_TX_WEIGHT / 1000 + counteparty's `dust_limit_satoshis`)
@@ -10413,13 +10394,13 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                if on_holder_tx {
                        let dust_outbound_overflow = dust_outbound_htlc_on_holder_tx_msat * (dust_outbound_htlc_on_holder_tx + 1);
                        let dust_inbound_overflow = dust_inbound_htlc_on_holder_tx_msat * dust_inbound_htlc_on_holder_tx + dust_outbound_htlc_on_holder_tx_msat;
-                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", if dust_outbound_balance { dust_outbound_overflow } else { dust_inbound_overflow }, config.channel_config.max_dust_htlc_exposure_msat)));
+                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", if dust_outbound_balance { dust_outbound_overflow } else { dust_inbound_overflow }, config.channel_config.max_dust_htlc_exposure_msat)));
                } else {
-                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", dust_overflow, config.channel_config.max_dust_htlc_exposure_msat)));
+                       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", dust_overflow, config.channel_config.max_dust_htlc_exposure_msat)));
                }
        } else if exposure_breach_event == ExposureEvent::AtHTLCReception {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], if on_holder_tx { dust_inbound_htlc_on_holder_tx_msat } else { dust_htlc_on_counterparty_tx_msat });
-               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+               nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[1], 1);
                let mut events = nodes[1].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -10437,7 +10418,7 @@ fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, exposure_breach_e
                }
        } else if exposure_breach_event == ExposureEvent::AtUpdateFeeOutbound {
                let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 2_500_000);
-               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) { panic!("Unexpected event at update_fee-swallowed HTLC", ); }
+               if let Err(_) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) { panic!("Unexpected event at update_fee-swallowed HTLC", ); }
                {
                        let mut feerate_lock = chanmon_cfgs[0].fee_estimator.sat_per_kw.lock().unwrap();
                        *feerate_lock = *feerate_lock * 10;
index c9ef10051e1261f848dd2e2d8c43eace347aa68e..731b68ce49f0d30365add7659e74792968872613 100644 (file)
@@ -14,7 +14,7 @@ use bitcoin::hashes::{Hash, HashEngine};
 use bitcoin::hashes::cmp::fixed_time_eq;
 use bitcoin::hashes::hmac::{Hmac, HmacEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
-use crate::chain::keysinterface::{KeyMaterial, KeysInterface, Sign};
+use crate::chain::keysinterface::{KeyMaterial, KeysInterface};
 use crate::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use crate::ln::msgs;
 use crate::ln::msgs::MAX_VALUE_MSAT;
@@ -91,8 +91,8 @@ impl Method {
 /// `current_time` is a Unix timestamp representing the current time.
 ///
 /// [phantom node payments]: crate::chain::keysinterface::PhantomKeysManager
-pub fn create<Signer: Sign, K: Deref>(keys: &ExpandedKey, min_value_msat: Option<u64>, invoice_expiry_delta_secs: u32, keys_manager: &K, current_time: u64) -> Result<(PaymentHash, PaymentSecret), ()>
-       where K::Target: KeysInterface<Signer = Signer>
+pub fn create<K: Deref>(keys: &ExpandedKey, min_value_msat: Option<u64>, invoice_expiry_delta_secs: u32, keys_manager: &K, current_time: u64) -> Result<(PaymentHash, PaymentSecret), ()>
+       where K::Target: KeysInterface
 {
        let metadata_bytes = construct_metadata_bytes(min_value_msat, Method::LdkPaymentHash, invoice_expiry_delta_secs, current_time)?;
 
index de54e3d12219009da9340f97dfa533190cb2112c..2916855c0ab34d27b1352ac57e594ad31b090cc9 100644 (file)
@@ -13,7 +13,7 @@ use crate::chain::channelmonitor::{ANTI_REORG_DELAY, Balance};
 use crate::chain::transaction::OutPoint;
 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
 use crate::ln::channel;
-use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT};
+use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT, PaymentId};
 use crate::ln::msgs::ChannelMessageHandler;
 use crate::util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
 
@@ -52,7 +52,7 @@ fn chanmon_fail_from_stale_commitment() {
        let (update_a, _, chan_id_2, _) = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let bs_txn = get_local_commitment_txn!(nodes[1], chan_id_2);
@@ -598,7 +598,7 @@ fn test_balances_on_local_commitment_htlcs() {
 
        let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 10_000_000);
        let htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -609,7 +609,7 @@ fn test_balances_on_local_commitment_htlcs() {
        expect_payment_received!(nodes[1], payment_hash, payment_secret, 10_000_000);
 
        let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[1], 20_000_000);
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
index 5833933951100a461b4ba6d38b0aa1ed25758111..1478f8b6bfb8272795dcd06d8272db926876a959 100644 (file)
@@ -42,7 +42,7 @@ use crate::io_extras::read_to_end;
 
 use crate::util::events::{MessageSendEventsProvider, OnionMessageProvider};
 use crate::util::logger;
-use crate::util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
+use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
 
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 
@@ -66,8 +66,7 @@ pub enum DecodeError {
        /// A length descriptor in the packet didn't describe the later data correctly
        BadLengthDescriptor,
        /// Error from std::io
-       Io(/// (C-not exported) as ErrorKind doesn't have a reasonable mapping
-        io::ErrorKind),
+       Io(io::ErrorKind),
        /// The message included zlib-compressed values, which we don't support.
        UnsupportedCompression,
 }
@@ -1028,9 +1027,6 @@ mod fuzzy_internal_msgs {
        }
 
        pub(crate) enum OnionHopDataFormat {
-               Legacy { // aka Realm-0
-                       short_channel_id: u64,
-               },
                NonFinalNode {
                        short_channel_id: u64,
                },
@@ -1046,7 +1042,6 @@ mod fuzzy_internal_msgs {
                /// Message serialization may panic if this value is more than 21 million Bitcoin.
                pub(crate) amt_to_forward: u64,
                pub(crate) outgoing_cltv_value: u32,
-               // 12 bytes of 0-padding for Legacy format
        }
 
        pub struct DecodedOnionErrorPacket {
@@ -1459,13 +1454,6 @@ impl Readable for FinalOnionHopData {
 impl Writeable for OnionHopData {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                match self.format {
-                       OnionHopDataFormat::Legacy { short_channel_id } => {
-                               0u8.write(w)?;
-                               short_channel_id.write(w)?;
-                               self.amt_to_forward.write(w)?;
-                               self.outgoing_cltv_value.write(w)?;
-                               w.write_all(&[0;12])?;
-                       },
                        OnionHopDataFormat::NonFinalNode { short_channel_id } => {
                                encode_varint_length_prefixed_tlv!(w, {
                                        (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
@@ -1488,58 +1476,44 @@ impl Writeable for OnionHopData {
 
 impl Readable for OnionHopData {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let b: BigSize = Readable::read(r)?;
-               const LEGACY_ONION_HOP_FLAG: u64 = 0;
-               let (format, amt, cltv_value) = if b.0 != LEGACY_ONION_HOP_FLAG {
-                       let mut rd = FixedLengthReader::new(r, b.0);
-                       let mut amt = HighZeroBytesDroppedBigSize(0u64);
-                       let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
-                       let mut short_id: Option<u64> = None;
-                       let mut payment_data: Option<FinalOnionHopData> = None;
-                       let mut keysend_preimage: Option<PaymentPreimage> = None;
-                       decode_tlv_stream!(&mut rd, {
-                               (2, amt, required),
-                               (4, cltv_value, required),
-                               (6, short_id, option),
-                               (8, payment_data, option),
-                               // See https://github.com/lightning/blips/blob/master/blip-0003.md
-                               (5482373484, keysend_preimage, option)
-                       });
-                       rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?;
-                       let format = if let Some(short_channel_id) = short_id {
-                               if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
-                               OnionHopDataFormat::NonFinalNode {
-                                       short_channel_id,
-                               }
-                       } else {
-                               if let &Some(ref data) = &payment_data {
-                                       if data.total_msat > MAX_VALUE_MSAT {
-                                               return Err(DecodeError::InvalidValue);
-                                       }
-                               }
-                               OnionHopDataFormat::FinalNode {
-                                       payment_data,
-                                       keysend_preimage,
-                               }
-                       };
-                       (format, amt.0, cltv_value.0)
+               let mut amt = HighZeroBytesDroppedBigSize(0u64);
+               let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
+               let mut short_id: Option<u64> = None;
+               let mut payment_data: Option<FinalOnionHopData> = None;
+               let mut keysend_preimage: Option<PaymentPreimage> = None;
+               read_tlv_fields!(r, {
+                       (2, amt, required),
+                       (4, cltv_value, required),
+                       (6, short_id, option),
+                       (8, payment_data, option),
+                       // See https://github.com/lightning/blips/blob/master/blip-0003.md
+                       (5482373484, keysend_preimage, option)
+               });
+
+               let format = if let Some(short_channel_id) = short_id {
+                       if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
+                       OnionHopDataFormat::NonFinalNode {
+                               short_channel_id,
+                       }
                } else {
-                       let format = OnionHopDataFormat::Legacy {
-                               short_channel_id: Readable::read(r)?,
-                       };
-                       let amt: u64 = Readable::read(r)?;
-                       let cltv_value: u32 = Readable::read(r)?;
-                       r.read_exact(&mut [0; 12])?;
-                       (format, amt, cltv_value)
+                       if let &Some(ref data) = &payment_data {
+                               if data.total_msat > MAX_VALUE_MSAT {
+                                       return Err(DecodeError::InvalidValue);
+                               }
+                       }
+                       OnionHopDataFormat::FinalNode {
+                               payment_data,
+                               keysend_preimage,
+                       }
                };
 
-               if amt > MAX_VALUE_MSAT {
+               if amt.0 > MAX_VALUE_MSAT {
                        return Err(DecodeError::InvalidValue);
                }
                Ok(OnionHopData {
                        format,
-                       amt_to_forward: amt,
-                       outgoing_cltv_value: cltv_value,
+                       amt_to_forward: amt.0,
+                       outgoing_cltv_value: cltv_value.0,
                })
        }
 }
@@ -2669,20 +2643,6 @@ mod tests {
                assert_eq!(encoded_value, target_value);
        }
 
-       #[test]
-       fn encoding_legacy_onion_hop_data() {
-               let msg = msgs::OnionHopData {
-                       format: OnionHopDataFormat::Legacy {
-                               short_channel_id: 0xdeadbeef1bad1dea,
-                       },
-                       amt_to_forward: 0x0badf00d01020304,
-                       outgoing_cltv_value: 0xffffffff,
-               };
-               let encoded_value = msg.encode();
-               let target_value = hex::decode("00deadbeef1bad1dea0badf00d01020304ffffffff000000000000000000000000").unwrap();
-               assert_eq!(encoded_value, target_value);
-       }
-
        #[test]
        fn encoding_nonfinal_onion_hop_data() {
                let mut msg = msgs::OnionHopData {
index 0b6012f637a52ad5090ff1ab782cabb202c9f872..b5c55ccf5ca0c60f945dc4b87c516c83ff27a6fa 100644 (file)
@@ -15,11 +15,11 @@ use crate::chain::channelmonitor::{ChannelMonitor, CLTV_CLAIM_BUFFER, LATENCY_GR
 use crate::chain::keysinterface::{KeysInterface, Recipient};
 use crate::ln::{PaymentHash, PaymentSecret};
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingHTLCInfo, PendingHTLCRouting};
+use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId};
 use crate::ln::onion_utils;
-use crate::routing::gossip::{NetworkUpdate, RoutingFees, NodeId};
+use crate::routing::gossip::{NetworkUpdate, RoutingFees};
 use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop};
-use crate::ln::features::{InitFeatures, InvoiceFeatures, NodeFeatures};
+use crate::ln::features::{InitFeatures, InvoiceFeatures};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, ChannelUpdate};
 use crate::ln::wire::Encode;
@@ -82,7 +82,8 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
        }
 
        // 0 ~~> 2 send payment
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(*payment_secret)).unwrap();
+       let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
+       nodes[0].node.send_payment(&route, *payment_hash, &Some(*payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        // temper update_add (0 => 1)
@@ -212,6 +213,16 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
        } else {
                panic!("Unexpected event");
        }
+       nodes[0].node.abandon_payment(payment_id);
+       let events = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 1);
+       match events[0] {
+               Event::PaymentFailed { payment_hash: ev_payment_hash, payment_id: ev_payment_id } => {
+                       assert_eq!(*payment_hash, ev_payment_hash);
+                       assert_eq!(payment_id, ev_payment_id);
+               }
+               _ => panic!("Unexpected second event"),
+       }
 }
 
 impl msgs::ChannelUpdate {
@@ -274,7 +285,7 @@ fn test_fee_failures() {
 
        // positive case
        let (route, payment_hash_success, payment_preimage_success, payment_secret_success) = get_route_and_payment_hash!(nodes[0], nodes[2], 40_000);
-       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
@@ -295,7 +306,7 @@ fn test_fee_failures() {
        }
 
        let (payment_preimage_success, payment_hash_success, payment_secret_success) = get_payment_preimage_hash!(nodes[2]);
-       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
@@ -344,7 +355,7 @@ fn test_onion_failure() {
                // break the first (non-final) hop payload by swapping the realm (0) byte for a byte
                // describing a length-1 TLV payload, which is obviously bogus.
                new_payloads[0].data[0] = 1;
-               msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
+               msg.onion_routing_packet = onion_utils::construct_onion_packet_with_writable_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
        }, ||{}, true, Some(PERM|22), Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent: true}), Some(short_channel_id));
 
        // final node failure
@@ -361,7 +372,7 @@ fn test_onion_failure() {
                // break the last-hop payload by swapping the realm (0) byte for a byte describing a
                // length-1 TLV payload, which is obviously bogus.
                new_payloads[1].data[0] = 1;
-               msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
+               msg.onion_routing_packet = onion_utils::construct_onion_packet_with_writable_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
        }, ||{}, false, Some(PERM|22), Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent: true}), Some(short_channel_id));
 
        // the following three with run_onion_failure_test_with_fail_intercept() test only the origin node
@@ -543,7 +554,7 @@ fn test_onion_failure() {
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
-                                       &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
+                                       &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
                                                forward_info.outgoing_cltv_value += 1,
                                        _ => {},
                                }
@@ -556,8 +567,8 @@ fn test_onion_failure() {
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
-                                       &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
-                                               forward_info.amt_to_forward -= 1,
+                                       &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
+                                               forward_info.outgoing_amt_msat -= 1,
                                        _ => {},
                                }
                        }
@@ -766,7 +777,7 @@ fn do_test_onion_failure_stale_channel_update(announced_channel: bool) {
                channel_monitors.insert(chanmon_1.get_funding_txo().0, &mut chanmon_1);
                channel_monitors.insert(chanmon_2.get_funding_txo().0, &mut chanmon_2);
 
-               let chanmgr = <(_, ChannelManager<_, _, _, _, _, _>)>::read(
+               let chanmgr = <(_, ChannelManager<_, _, _, _, _>)>::read(
                        &mut &nodes[1].node.encode()[..], ChannelManagerReadArgs {
                                default_config: *nodes[1].node.get_current_default_configuration(),
                                keys_manager: nodes[1].keys_manager,
@@ -791,166 +802,52 @@ fn test_onion_failure_stale_channel_update() {
 }
 
 #[test]
-fn test_default_to_onion_payload_tlv_format() {
-       // Tests that we default to creating tlv format onion payloads when no `NodeAnnouncementInfo`
-       // `features` for a node in the `network_graph` exists, or when the node isn't in the
-       // `network_graph`, and no other known `features` for the node exists.
-       let mut priv_channels_conf = UserConfig::default();
-       priv_channels_conf.channel_handshake_config.announced_channel = false;
-       let chanmon_cfgs = create_chanmon_cfgs(5);
-       let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &[None, None, None, None, Some(priv_channels_conf)]);
-       let mut nodes = create_network(5, &node_cfgs, &node_chanmgrs);
-
-       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 2, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_unannounced_chan_between_nodes_with_value(&nodes, 3, 4, 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-
-       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id());
-       let origin_node = &nodes[0];
-       let network_graph = origin_node.network_graph;
-
-       // Clears all the `NodeAnnouncementInfo` for all nodes of `nodes[0]`'s `network_graph`, so that
-       // their `features` aren't used when creating the `route`.
-       network_graph.clear_nodes_announcement_info();
-
-       let (announced_route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[3], payment_params, 10_000, TEST_FINAL_CLTV);
-
-       let hops = &announced_route.paths[0];
-       // Assert that the hop between `nodes[1]` and `nodes[2]` defaults to supporting variable length
-       // onions, as `nodes[0]` has no `NodeAnnouncementInfo` `features` for `node[2]`
-       assert!(hops[1].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[2]` and `nodes[3]` defaults to supporting variable length
-       // onions, as `nodes[0]` has no `NodeAnnouncementInfo` `features` for `node[3]`, and no `InvoiceFeatures`
-       // for the `payment_params`, which would otherwise have been used.
-       assert!(hops[2].node_features.supports_variable_length_onion());
-       // Note that we do not assert that `hops[0]` (the channel between `nodes[0]` and `nodes[1]`)
-       // supports variable length onions, as the `InitFeatures` exchanged in the init message
-       // between the nodes will be used when creating the route. We therefore do not default to
-       // supporting variable length onions for that hop, as the `InitFeatures` in this case are
-       // `channelmanager::provided_init_features()`.
-
-       let unannounced_chan = &nodes[4].node.list_usable_channels()[0];
-
-       let last_hop = RouteHint(vec![RouteHintHop {
-               src_node_id: nodes[3].node.get_our_node_id(),
-               short_channel_id: unannounced_chan.short_channel_id.unwrap(),
-               fees: RoutingFees {
-                       base_msat: 0,
-                       proportional_millionths: 0,
-               },
-               cltv_expiry_delta: 42,
-               htlc_minimum_msat: None,
-               htlc_maximum_msat: None,
-       }]);
-
-       let unannounced_chan_params = PaymentParameters::from_node_id(nodes[4].node.get_our_node_id()).with_route_hints(vec![last_hop]);
-       let (unannounced_route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[4], unannounced_chan_params, 10_000, TEST_FINAL_CLTV);
-
-       let unannounced_chan_hop = &unannounced_route.paths[0][3];
-       // Ensure that `nodes[4]` doesn't exist in `nodes[0]`'s `network_graph`, as it's not public.
-       assert!(&network_graph.read_only().nodes().get(&NodeId::from_pubkey(&nodes[4].node.get_our_node_id())).is_none());
-       // Assert that the hop between `nodes[3]` and `nodes[4]` defaults to supporting variable length
-       // onions, even though `nodes[4]` as `nodes[0]` doesn't exists in `nodes[0]`'s `network_graph`,
-       // and no `InvoiceFeatures` for the `payment_params` exists, which would otherwise have been
-       // used.
-       assert!(unannounced_chan_hop.node_features.supports_variable_length_onion());
-
-       let cur_height = nodes[0].best_block_info().1 + 1;
-       let (announced_route_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&announced_route.paths[0], 40000, &None, cur_height, &None).unwrap();
-       let (unannounced_route_paylods, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&unannounced_route.paths[0], 40000, &None, cur_height, &None).unwrap();
-
-       for onion_payloads in vec![announced_route_payloads, unannounced_route_paylods] {
-               for onion_payload in onion_payloads.iter() {
-                       match onion_payload.format {
-                               msgs::OnionHopDataFormat::Legacy {..} => {
-                                       panic!("Generated a `msgs::OnionHopDataFormat::Legacy` payload, even though that shouldn't have happend.");
-                               }
-                               _ => {}
-                       }
-               }
-       }
-}
+fn test_always_create_tlv_format_onion_payloads() {
+       // Verify that we always generate tlv onion format payloads, even if the features specifically
+       // specifies no support for variable length onions, as the legacy payload format has been
+       // deprecated in BOLT4.
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let mut node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 
-#[test]
-fn test_do_not_default_to_onion_payload_tlv_format_when_unsupported() {
-       // Tests that we do not default to creating tlv onions if either of these types features
-       // exists, which specifies no support for variable length onions for a specific hop, when
-       // creating a route:
-       // 1. `InitFeatures` to the counterparty node exchanged with the init message to the node.
-       // 2. `NodeFeatures` in the `NodeAnnouncementInfo` of a node in sender node's `network_graph`.
-       // 3. `InvoiceFeatures` specified by the receiving node, when no `NodeAnnouncementInfo`
-       // `features` exists for the receiver in the sender's `network_graph`.
-       let chanmon_cfgs = create_chanmon_cfgs(4);
-       let mut node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
-
-       // Set `node[1]` config to `InitFeatures::empty()` + `static_remote_key` which implies
-       // `!supports_variable_length_onion()` but still supports the required static-remote-key
-       // feature.
+       // Set `node[1]`'s config features to features which return `false` for
+       // `supports_variable_length_onion()`
+       let mut no_variable_length_onion_features = InitFeatures::empty();
+       no_variable_length_onion_features.set_static_remote_key_required();
        let mut node_1_cfg = &mut node_cfgs[1];
-       node_1_cfg.features = InitFeatures::empty();
-       node_1_cfg.features.set_static_remote_key_required();
+       node_1_cfg.features = no_variable_length_onion_features;
 
-       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
-       let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+       let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
-       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 2, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+       create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::empty(), InitFeatures::empty());
+       create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::empty(), InitFeatures::empty());
 
-       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id())
+       let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id())
                .with_features(InvoiceFeatures::empty());
-       let origin_node = &nodes[0];
-       let network_graph = origin_node.network_graph;
-       network_graph.clear_nodes_announcement_info();
-
-       // Set `NodeAnnouncementInfo` `features` which do not support variable length onions for
-       // `nodes[2]` in `nodes[0]`'s `network_graph`.
-       let nodes_2_unsigned_node_announcement = msgs::UnsignedNodeAnnouncement {
-               features: NodeFeatures::empty(),
-               timestamp: 0,
-               node_id: nodes[2].node.get_our_node_id(),
-               rgb: [32; 3],
-               alias: [16;32],
-               addresses: Vec::new(),
-               excess_address_data: Vec::new(),
-               excess_data: Vec::new(),
-       };
-       let _res = network_graph.update_node_from_unsigned_announcement(&nodes_2_unsigned_node_announcement);
-
-       let (route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[3], payment_params, 10_000, TEST_FINAL_CLTV);
+       let (route, _payment_hash, _payment_preimage, _payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 40000, TEST_FINAL_CLTV);
 
        let hops = &route.paths[0];
-
-       // Assert that the hop between `nodes[0]` and `nodes[1]` doesn't support variable length
-       // onions, as as the `InitFeatures` exchanged (`InitFeatures::empty()`) in the init message
-       // between the nodes when setting up the channel is used when creating the `route` and that we
-       // therefore do not default to supporting variable length onions. Despite `nodes[0]` having no
-       // `NodeAnnouncementInfo` `features` for `node[1]`.
+       // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[0].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[1]` and `nodes[2]` uses the `features` from
-       // `nodes_2_unsigned_node_announcement` that doesn't support variable length onions.
+       // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[1].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[2]` and `nodes[3]` uses the `InvoiceFeatures` set to the
-       // `payment_params`, that doesn't support variable length onions. We therefore do not end up
-       // defaulting to supporting variable length onions, despite `nodes[0]` having no
-       // `NodeAnnouncementInfo` `features` for `node[3]`.
-       assert!(!hops[2].node_features.supports_variable_length_onion());
 
        let cur_height = nodes[0].best_block_info().1 + 1;
        let (onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, cur_height, &None).unwrap();
 
-       for onion_payload in onion_payloads.iter() {
-               match onion_payload.format {
-                       msgs::OnionHopDataFormat::Legacy {..} => {}
-                       _ => {
-                               panic!("Should have only have generated `msgs::OnionHopDataFormat::Legacy` payloads");
-                       }
-               }
+       match onion_payloads[0].format {
+               msgs::OnionHopDataFormat::NonFinalNode {..} => {},
+               _ => { panic!(
+                       "Should have generated a `msgs::OnionHopDataFormat::NonFinalNode` payload for `hops[0]`,
+                       despite that the features signals no support for variable length onions"
+               )}
+       }
+       match onion_payloads[1].format {
+               msgs::OnionHopDataFormat::FinalNode {..} => {},
+               _ => {panic!(
+                       "Should have generated a `msgs::OnionHopDataFormat::FinalNode` payload for `hops[1]`,
+                       despite that the features signals no support for variable length onions"
+               )}
        }
 }
 
@@ -1011,7 +908,7 @@ fn test_phantom_onion_hmac_failure() {
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1024,12 +921,12 @@ fn test_phantom_onion_hmac_failure() {
                let mut forward_htlcs = nodes[1].node.forward_htlcs.lock().unwrap();
                let mut pending_forward = forward_htlcs.get_mut(&phantom_scid).unwrap();
                match pending_forward[0] {
-                       HTLCForwardInfo::AddHTLC {
+                       HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                forward_info: PendingHTLCInfo {
                                        routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                        ..
                                }, ..
-                       } => {
+                       }) => {
                                onion_packet.hmac[onion_packet.hmac.len() - 1] ^= 1;
                                Sha256::hash(&onion_packet.hop_data).into_inner().to_vec()
                        },
@@ -1072,7 +969,7 @@ fn test_phantom_invalid_onion_payload() {
        // We'll use the session priv later when constructing an invalid onion packet.
        let session_priv = [3; 32];
        *nodes[0].keys_manager.override_random_bytes.lock().unwrap() = Some(session_priv);
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1084,12 +981,12 @@ fn test_phantom_invalid_onion_payload() {
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
-                               &mut HTLCForwardInfo::AddHTLC {
+                               &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo {
                                                routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                                ..
                                        }, ..
-                               } => {
+                               }) => {
                                        // Construct the onion payloads for the entire route and an invalid amount.
                                        let height = nodes[0].best_block_info().1;
                                        let session_priv = SecretKey::from_slice(&session_priv).unwrap();
@@ -1143,7 +1040,7 @@ fn test_phantom_final_incorrect_cltv_expiry() {
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1155,9 +1052,9 @@ fn test_phantom_final_incorrect_cltv_expiry() {
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
-                               &mut HTLCForwardInfo::AddHTLC {
+                               &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo { ref mut outgoing_cltv_value, .. }, ..
-                               } => {
+                               }) => {
                                        *outgoing_cltv_value += 1;
                                },
                                _ => panic!("Unexpected forward"),
@@ -1202,7 +1099,7 @@ fn test_phantom_failure_too_low_cltv() {
        route.paths[0][1].cltv_expiry_delta = 5;
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1245,7 +1142,7 @@ fn test_phantom_failure_too_low_recv_amt() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, bad_recv_amt_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1297,7 +1194,7 @@ fn test_phantom_dust_exposure_failure() {
        let (mut route, _) = get_phantom_route!(nodes, max_dust_exposure + 1, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@ -1340,7 +1237,7 @@ fn test_phantom_failure_reject_payment() {
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_amt_msat, channel);
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
index 598edcf0367952b011e290bfe48938ec92364ecb..23dc556cfacca4f129d7968323f997a0f3c63692 100644 (file)
@@ -82,6 +82,15 @@ pub(super) fn gen_ammag_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] {
        Hmac::from_engine(hmac).into_inner()
 }
 
+#[cfg(test)]
+#[inline]
+pub(super) fn gen_pad_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] {
+       assert_eq!(shared_secret.len(), 32);
+       let mut hmac = HmacEngine::<Sha256>::new(&[0x70, 0x61, 0x64]); // pad
+       hmac.input(&shared_secret);
+       Hmac::from_engine(hmac).into_inner()
+}
+
 pub(crate) fn next_hop_packet_pubkey<T: secp256k1::Signing + secp256k1::Verification>(secp_ctx: &Secp256k1<T>, packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result<PublicKey, secp256k1::Error> {
        let blinding_factor = {
                let mut sha = Sha256::engine();
@@ -153,24 +162,18 @@ pub(super) fn build_onion_payloads(path: &Vec<RouteHop>, total_msat: u64, paymen
                let value_msat = if cur_value_msat == 0 { hop.fee_msat } else { cur_value_msat };
                let cltv = if cur_cltv == starting_htlc_offset { hop.cltv_expiry_delta + starting_htlc_offset } else { cur_cltv };
                res.insert(0, msgs::OnionHopData {
-                       format: if hop.node_features.supports_variable_length_onion() {
-                               if idx == 0 {
-                                       msgs::OnionHopDataFormat::FinalNode {
-                                               payment_data: if let &Some(ref payment_secret) = payment_secret_option {
-                                                       Some(msgs::FinalOnionHopData {
-                                                               payment_secret: payment_secret.clone(),
-                                                               total_msat,
-                                                       })
-                                               } else { None },
-                                               keysend_preimage: *keysend_preimage,
-                                       }
-                               } else {
-                                       msgs::OnionHopDataFormat::NonFinalNode {
-                                               short_channel_id: last_short_channel_id,
-                                       }
+                       format: if idx == 0 {
+                               msgs::OnionHopDataFormat::FinalNode {
+                                       payment_data: if let &Some(ref payment_secret) = payment_secret_option {
+                                               Some(msgs::FinalOnionHopData {
+                                                       payment_secret: payment_secret.clone(),
+                                                       total_msat,
+                                               })
+                                       } else { None },
+                                       keysend_preimage: *keysend_preimage,
                                }
                        } else {
-                               msgs::OnionHopDataFormat::Legacy {
+                               msgs::OnionHopDataFormat::NonFinalNode {
                                        short_channel_id: last_short_channel_id,
                                }
                        },
@@ -230,9 +233,9 @@ pub(super) fn construct_onion_packet(payloads: Vec<msgs::OnionHopData>, onion_ke
 }
 
 #[cfg(test)]
-// Used in testing to write bogus OnionHopDatas, which is otherwise not representable in
-// msgs::OnionHopData.
-pub(super) fn construct_onion_packet_bogus_hopdata<HD: Writeable>(payloads: Vec<HD>, onion_keys: Vec<OnionKeys>, prng_seed: [u8; 32], associated_data: &PaymentHash) -> msgs::OnionPacket {
+/// Used in testing to write bogus `BogusOnionHopData` as well as `RawOnionHopData`, which is
+/// otherwise not representable in `msgs::OnionHopData`.
+pub(super) fn construct_onion_packet_with_writable_hopdata<HD: Writeable>(payloads: Vec<HD>, onion_keys: Vec<OnionKeys>, prng_seed: [u8; 32], associated_data: &PaymentHash) -> msgs::OnionPacket {
        let mut packet_data = [0; ONION_DATA_LEN];
 
        let mut chacha = ChaCha20::new(&prng_seed, &[0; 8]);
@@ -589,31 +592,6 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
        } else { unreachable!(); }
 }
 
-/// An input used when decoding an onion packet.
-pub(crate) trait DecodeInput {
-       type Arg;
-       /// If Some, this is the input when checking the hmac of the onion packet.
-       fn payment_hash(&self) -> Option<&PaymentHash>;
-       /// Read argument when decrypting our hop payload.
-       fn read_arg(self) -> Self::Arg;
-}
-
-impl DecodeInput for PaymentHash {
-       type Arg = ();
-       fn payment_hash(&self) -> Option<&PaymentHash> {
-               Some(self)
-       }
-       fn read_arg(self) -> Self::Arg { () }
-}
-
-impl DecodeInput for SharedSecret {
-       type Arg = SharedSecret;
-       fn payment_hash(&self) -> Option<&PaymentHash> {
-               None
-       }
-       fn read_arg(self) -> Self::Arg { self }
-}
-
 /// Allows `decode_next_hop` to return the next hop packet bytes for either payments or onion
 /// message forwards.
 pub(crate) trait NextPacketBytes: AsMut<[u8]> {
@@ -664,7 +642,7 @@ pub(crate) enum OnionDecodeErr {
 }
 
 pub(crate) fn decode_next_payment_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: PaymentHash) -> Result<Hop, OnionDecodeErr> {
-       match decode_next_hop(shared_secret, hop_data, hmac_bytes, payment_hash) {
+       match decode_next_hop(shared_secret, hop_data, hmac_bytes, Some(payment_hash), ()) {
                Ok((next_hop_data, None)) => Ok(Hop::Receive(next_hop_data)),
                Ok((next_hop_data, Some((next_hop_hmac, FixedSizeOnionPacket(new_packet_bytes))))) => {
                        Ok(Hop::Forward {
@@ -677,12 +655,16 @@ pub(crate) fn decode_next_payment_hop(shared_secret: [u8; 32], hop_data: &[u8],
        }
 }
 
-pub(crate) fn decode_next_hop<D: DecodeInput, R: ReadableArgs<D::Arg>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], decode_input: D) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
+pub(crate) fn decode_next_untagged_hop<T, R: ReadableArgs<T>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], read_args: T) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
+       decode_next_hop(shared_secret, hop_data, hmac_bytes, None, read_args)
+}
+
+fn decode_next_hop<T, R: ReadableArgs<T>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: Option<PaymentHash>, read_args: T) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
        let (rho, mu) = gen_rho_mu_from_shared_secret(&shared_secret);
        let mut hmac = HmacEngine::<Sha256>::new(&mu);
        hmac.input(hop_data);
-       if let Some(payment_hash) = decode_input.payment_hash() {
-               hmac.input(&payment_hash.0[..]);
+       if let Some(tag) = payment_hash {
+               hmac.input(&tag.0[..]);
        }
        if !fixed_time_eq(&Hmac::from_engine(hmac).into_inner(), &hmac_bytes) {
                return Err(OnionDecodeErr::Malformed {
@@ -693,7 +675,7 @@ pub(crate) fn decode_next_hop<D: DecodeInput, R: ReadableArgs<D::Arg>, N: NextPa
 
        let mut chacha = ChaCha20::new(&rho, &[0u8; 8]);
        let mut chacha_stream = ChaChaReader { chacha: &mut chacha, read: Cursor::new(&hop_data[..]) };
-       match R::read(&mut chacha_stream, decode_input.read_arg()) {
+       match R::read(&mut chacha_stream, read_args) {
                Err(err) => {
                        let error_code = match err {
                                msgs::DecodeError::UnknownVersion => 0x4000 | 1, // unknown realm byte
@@ -762,7 +744,7 @@ mod tests {
        use crate::ln::features::{ChannelFeatures, NodeFeatures};
        use crate::routing::router::{Route, RouteHop};
        use crate::ln::msgs;
-       use crate::util::ser::{Writeable, Writer};
+       use crate::util::ser::{Writeable, Writer, VecWriter};
 
        use hex;
 
@@ -771,6 +753,10 @@ mod tests {
 
        use super::OnionKeys;
 
+       fn get_test_session_key() -> SecretKey {
+               SecretKey::from_slice(&hex::decode("4141414141414141414141414141414141414141414141414141414141414141").unwrap()[..]).unwrap()
+       }
+
        fn build_test_onion_keys() -> Vec<OnionKeys> {
                // Keys from BOLT 4, used in both test vector tests
                let secp_ctx = Secp256k1::new();
@@ -780,44 +766,43 @@ mod tests {
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                        ]],
                        payment_params: None,
                };
 
-               let session_priv = SecretKey::from_slice(&hex::decode("4141414141414141414141414141414141414141414141414141414141414141").unwrap()[..]).unwrap();
-
-               let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
+               let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &get_test_session_key()).unwrap();
                assert_eq!(onion_keys.len(), route.paths[0].len());
                onion_keys
        }
 
        #[test]
        fn onion_vectors() {
-               // Legacy packet creation test vectors from BOLT 4
                let onion_keys = build_test_onion_keys();
 
+               // Test generation of ephemeral keys and secrets. These values used to be part of the BOLT4
+               // test vectors, but have since been removed. We keep them as they provide test coverage.
                assert_eq!(onion_keys[0].shared_secret.secret_bytes(), hex::decode("53eb63ea8a3fec3b3cd433b85cd62a4b145e1dda09391b348c4e1cd36a03ea66").unwrap()[..]);
                assert_eq!(onion_keys[0].blinding_factor[..], hex::decode("2ec2e5da605776054187180343287683aa6a51b4b1c04d6dd49c45d8cffb3c36").unwrap()[..]);
                assert_eq!(onion_keys[0].ephemeral_pubkey.serialize()[..], hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]);
@@ -848,49 +833,95 @@ mod tests {
                assert_eq!(onion_keys[4].rho, hex::decode("034e18b8cc718e8af6339106e706c52d8df89e2b1f7e9142d996acf88df8799b").unwrap()[..]);
                assert_eq!(onion_keys[4].mu, hex::decode("8e45e5c61c2b24cb6382444db6698727afb063adecd72aada233d4bf273d975a").unwrap()[..]);
 
-               // Test vectors below are flat-out wrong: they claim to set outgoing_cltv_value to non-0 :/
+               // Packet creation test vectors from BOLT 4 (see
+               // https://github.com/lightning/bolts/blob/16973e2b857e853308cafd59e42fa830d75b1642/bolt04/onion-test.json).
+               // Note that we represent the test vector payloads 2 and 5 through RawOnionHopData::data
+               // with raw hex instead of our in-memory enums, as the payloads contains custom types, and
+               // we have no way of representing that with our enums.
                let payloads = vec!(
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0,
-                               },
-                               amt_to_forward: 0,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0101010101010101,
-                               },
-                               amt_to_forward: 0x0100000001,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0202020202020202,
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 1,
                                },
-                               amt_to_forward: 0x0200000002,
-                               outgoing_cltv_value: 0,
+                               amt_to_forward: 15000,
+                               outgoing_cltv_value: 1500,
+                       }),
+                       /*
+                       The second payload is represented by raw hex as it contains custom type data. Content:
+                       1. length "52" (payload_length 82).
+
+                       The first part of the payload has the `NonFinalNode` format, with content as follows:
+                       2. amt_to_forward "020236b0"
+                          02 (type amt_to_forward) 02 (length 2) 36b0 (value 14000)
+                       3. outgoing_cltv_value "04020578"
+                          04 (type outgoing_cltv_value) 02 (length 2) 0578 (value 1400)
+                       4. short_channel_id "06080000000000000002"
+                          06 (type short_channel_id) 08 (length 8) 0000000000000002 (value 2)
+
+                       The rest of the payload is custom type data:
+                       5. custom_record "fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f"
+                       */
+                       RawOnionHopData {
+                               data: hex::decode("52020236b00402057806080000000000000002fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f").unwrap(),
                        },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0303030303030303,
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 3,
                                },
-                               amt_to_forward: 0x0300000003,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0404040404040404,
+                               amt_to_forward: 12500,
+                               outgoing_cltv_value: 1250,
+                       }),
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 4,
                                },
-                               amt_to_forward: 0x0400000004,
-                               outgoing_cltv_value: 0,
+                               amt_to_forward: 10000,
+                               outgoing_cltv_value: 1000,
+                       }),
+                       /*
+                       The fifth payload is represented by raw hex as it contains custom type data. Content:
+                       1. length "fd0110" (payload_length 272).
+
+                       The first part of the payload has the `FinalNode` format, with content as follows:
+                       1. amt_to_forward "02022710"
+                          02 (type amt_to_forward) 02 (length 2) 2710 (value 10000)
+                       2. outgoing_cltv_value "040203e8"
+                          04 (type outgoing_cltv_value) 02 (length 2) 03e8 (value 1000)
+                       3. payment_data "082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710"
+                          08 (type short_channel_id) 22 (length 34) 24a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f61704 (payment_secret) 2710 (total_msat value 10000)
+
+                       The rest of the payload is custom type data:
+                       4. custom_record "fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a"
+                       */
+                       RawOnionHopData {
+                               data: hex::decode("fd011002022710040203e8082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a").unwrap(),
                        },
                );
 
-               let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32])));
-               // Just check the final packet encoding, as it includes all the per-hop vectors in it
-               // anyway...
-               assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a716a996c7845c93d90e4ecbb9bde4ece2f69425c99e4bc820e44485455f135edc0d10f7d61ab590531cf08000179a333a347f8b4072f216400406bdf3bf038659793d4a1fd7b246979e3150a0a4cb052c9ec69acf0f48c3d39cd55675fe717cb7d80ce721caad69320c3a469a202f1e468c67eaf7a7cd8226d0fd32f7b48084dca885d56047694762b67021713ca673929c163ec36e04e40ca8e1c6d17569419d3039d9a1ec866abe044a9ad635778b961fc0776dc832b3a451bd5d35072d2269cf9b040f6b7a7dad84fb114ed413b1426cb96ceaf83825665ed5a1d002c1687f92465b49ed4c7f0218ff8c6c7dd7221d589c65b3b9aaa71a41484b122846c7c7b57e02e679ea8469b70e14fe4f70fee4d87b910cf144be6fe48eef24da475c0b0bcc6565ae82cd3f4e3b24c76eaa5616c6111343306ab35c1fe5ca4a77c0e314ed7dba39d6f1e0de791719c241a939cc493bea2bae1c1e932679ea94d29084278513c77b899cc98059d06a27d171b0dbdf6bee13ddc4fc17a0c4d2827d488436b57baa167544138ca2e64a11b43ac8a06cd0c2fba2d4d900ed2d9205305e2d7383cc98dacb078133de5f6fb6bed2ef26ba92cea28aafc3b9948dd9ae5559e8bd6920b8cea462aa445ca6a95e0e7ba52961b181c79e73bd581821df2b10173727a810c92b83b5ba4a0403eb710d2ca10689a35bec6c3a708e9e92f7d78ff3c5d9989574b00c6736f84c199256e76e19e78f0c98a9d580b4a658c84fc8f2096c2fbea8f5f8c59d0fdacb3be2802ef802abbecb3aba4acaac69a0e965abd8981e9896b1f6ef9d60f7a164b371af869fd0e48073742825e9434fc54da837e120266d53302954843538ea7c6c3dbfb4ff3b2fdbe244437f2a153ccf7bdb4c92aa08102d4f3cff2ae5ef86fab4653595e6a5837fa2f3e29f27a9cde5966843fb847a4a61f1e76c281fe8bb2b0a181d096100db5a1a5ce7a910238251a43ca556712eaadea167fb4d7d75825e440f3ecd782036d7574df8bceacb397abefc5f5254d2722215c53ff54af8299aaaad642c6d72a14d27882d9bbd539e1cc7a527526ba89b8c037ad09120e98ab042d3e8652b31ae0e478516bfaf88efca9f3676ffe99d2819dcaeb7610a626695f53117665d267d3f7abebd6bbd6733f645c72c389f03855bdf1e4b8075b516569b118233a0f0971d24b83113c0b096f5216a207ca99a7cddc81c130923fe3d91e7508c9ac5f2e914ff5dccab9e558566fa14efb34ac98d878580814b94b73acbfde9072f30b881f7f0fff42d4045d1ace6322d86a97d164aa84d93a60498065cc7c20e636f5862dc81531a88c60305a2e59a985be327a6902e4bed986dbf4a0b50c217af0ea7fdf9ab37f9ea1a1aaa72f54cf40154ea9b269f1a7c09f9f43245109431a175d50e2db0132337baa0ef97eed0fcf20489da36b79a1172faccc2f7ded7c60e00694282d93359c4682135642bc81f433574aa8ef0c97b4ade7ca372c5ffc23c7eddd839bab4e0f14d6df15c9dbeab176bec8b5701cf054eb3072f6dadc98f88819042bf10c407516ee58bce33fbe3b3d86a54255e577db4598e30a135361528c101683a5fcde7e8ba53f3456254be8f45fe3a56120ae96ea3773631fcb3873aa3abd91bcff00bd38bd43697a2e789e00da6077482e7b1b1a677b5afae4c54e6cbdf7377b694eb7d7a5b913476a5be923322d3de06060fd5e819635232a2cf4f0731da13b8546d1d6d4f8d75b9fce6c2341a71b0ea6f780df54bfdb0dd5cd9855179f602f9172307c7268724c3618e6817abd793adc214a0dc0bc616816632f27ea336fb56dfd").unwrap());
+               // Verify that the serialized OnionHopDataFormat::NonFinalNode tlv payloads matches the test vectors
+               let mut w = VecWriter(Vec::new());
+               payloads[0].write(&mut w).unwrap();
+               let hop_1_serialized_payload = w.0;
+               let expected_serialized_hop_1_payload = &hex::decode("1202023a98040205dc06080000000000000001").unwrap()[..];
+               assert_eq!(hop_1_serialized_payload, expected_serialized_hop_1_payload);
+
+               w = VecWriter(Vec::new());
+               payloads[2].write(&mut w).unwrap();
+               let hop_3_serialized_payload = w.0;
+               let expected_serialized_hop_3_payload = &hex::decode("12020230d4040204e206080000000000000003").unwrap()[..];
+               assert_eq!(hop_3_serialized_payload, expected_serialized_hop_3_payload);
+
+               w = VecWriter(Vec::new());
+               payloads[3].write(&mut w).unwrap();
+               let hop_4_serialized_payload = w.0;
+               let expected_serialized_hop_4_payload = &hex::decode("1202022710040203e806080000000000000004").unwrap()[..];
+               assert_eq!(hop_4_serialized_payload, expected_serialized_hop_4_payload);
+
+               let pad_keytype_seed = super::gen_pad_from_shared_secret(&get_test_session_key().secret_bytes());
+
+               let packet: msgs::OnionPacket = super::construct_onion_packet_with_writable_hopdata::<_>(payloads, onion_keys, pad_keytype_seed, &PaymentHash([0x42; 32]));
+
+               assert_eq!(packet.encode(), hex::decode("0002EEC7245D6B7D2CCB30380BFBE2A3648CD7A942653F5AA340EDCEA1F283686619F7F3416A5AA36DC7EEB3EC6D421E9615471AB870A33AC07FA5D5A51DF0A8823AABE3FEA3F90D387529D4F72837F9E687230371CCD8D263072206DBED0234F6505E21E282ABD8C0E4F5B9FF8042800BBAB065036EADD0149B37F27DDE664725A49866E052E809D2B0198AB9610FAA656BBF4EC516763A59F8F42C171B179166BA38958D4F51B39B3E98706E2D14A2DAFD6A5DF808093ABFCA5AEAACA16EDED5DB7D21FB0294DD1A163EDF0FB445D5C8D7D688D6DD9C541762BF5A5123BF9939D957FE648416E88F1B0928BFA034982B22548E1A4D922690EECF546275AFB233ACF4323974680779F1A964CFE687456035CC0FBA8A5428430B390F0057B6D1FE9A8875BFA89693EEB838CE59F09D207A503EE6F6299C92D6361BC335FCBF9B5CD44747AADCE2CE6069CFDC3D671DAEF9F8AE590CF93D957C9E873E9A1BC62D9640DC8FC39C14902D49A1C80239B6C5B7FD91D05878CBF5FFC7DB2569F47C43D6C0D27C438ABFF276E87364DEB8858A37E5A62C446AF95D8B786EAF0B5FCF78D98B41496794F8DCAAC4EEF34B2ACFB94C7E8C32A9E9866A8FA0B6F2A06F00A1CCDE569F97EEC05C803BA7500ACC96691D8898D73D8E6A47B8F43C3D5DE74458D20EDA61474C426359677001FBD75A74D7D5DB6CB4FEB83122F133206203E4E2D293F838BF8C8B3A29ACB321315100B87E80E0EDB272EE80FDA944E3FB6084ED4D7F7C7D21C69D9DA43D31A90B70693F9B0CC3EAC74C11AB8FF655905688916CFA4EF0BD04135F2E50B7C689A21D04E8E981E74C6058188B9B1F9DFC3EEC6838E9FFBCF22CE738D8A177C19318DFFEF090CEE67E12DE1A3E2A39F61247547BA5257489CBC11D7D91ED34617FCC42F7A9DA2E3CF31A94A210A1018143173913C38F60E62B24BF0D7518F38B5BAB3E6A1F8AEB35E31D6442C8ABB5178EFC892D2E787D79C6AD9E2FC271792983FA9955AC4D1D84A36C024071BC6E431B625519D556AF38185601F70E29035EA6A09C8B676C9D88CF7E05E0F17098B584C4168735940263F940033A220F40BE4C85344128B14BEB9E75696DB37014107801A59B13E89CD9D2258C169D523BE6D31552C44C82FF4BB18EC9F099F3BF0E5B1BB2BA9A87D7E26F98D294927B600B5529C47E04D98956677CBCEE8FA2B60F49776D8B8C367465B7C626DA53700684FB6C918EAD0EAB8360E4F60EDD25B4F43816A75ECF70F909301825B512469F8389D79402311D8AECB7B3EF8599E79485A4388D87744D899F7C47EE644361E17040A7958C8911BE6F463AB6A9B2AFACD688EC55EF517B38F1339EFC54487232798BB25522FF4572FF68567FE830F92F7B8113EFCE3E98C3FFFBAEDCE4FD8B50E41DA97C0C08E423A72689CC68E68F752A5E3A9003E64E35C957CA2E1C48BB6F64B05F56B70B575AD2F278D57850A7AD568C24A4D32A3D74B29F03DC125488BC7C637DA582357F40B0A52D16B3B40BB2C2315D03360BC24209E20972C200566BCF3BBE5C5B0AEDD83132A8A4D5B4242BA370B6D67D9B67EB01052D132C7866B9CB502E44796D9D356E4E3CB47CC527322CD24976FE7C9257A2864151A38E568EF7A79F10D6EF27CC04CE382347A2488B1F404FDBF407FE1CA1C9D0D5649E34800E25E18951C98CAE9F43555EEF65FEE1EA8F15828807366C3B612CD5753BF9FB8FCED08855F742CDDD6F765F74254F03186683D646E6F09AC2805586C7CF11998357CAFC5DF3F285329366F475130C928B2DCEBA4AA383758E7A9D20705C4BB9DB619E2992F608A1BA65DB254BB389468741D0502E2588AEB54390AC600C19AF5C8E61383FC1BEBE0029E4474051E4EF908828DB9CCA13277EF65DB3FD47CCC2179126AAEFB627719F421E20").unwrap());
        }
 
        #[test]
@@ -930,45 +961,4 @@ mod tests {
                        writer.write_all(&self.data[..])
                }
        }
-
-       #[test]
-       fn variable_length_onion_vectors() {
-               // Packet creation test vectors from BOLT 4 (as of this writing at
-               // bolt04/onion-test-multi-frame.json in the spec repo).
-               // Note that we use he RawOnionHopData for everything except Legacy hops, as even the hops
-               // with "type": "tlv" are not valid TLV (they were for a previous version of TLV that
-               // didn't move forward), and, thus, cannot be directly represented in our in-memory enums.
-               let onion_keys = build_test_onion_keys();
-
-               let payloads = vec!(
-                       RawOnionHopData::new(msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0,
-                               },
-                               amt_to_forward: 0,
-                               outgoing_cltv_value: 0,
-                       }),
-                       RawOnionHopData {
-                               data: hex::decode("140101010101010101000000000000000100000001").unwrap(),
-                       },
-                       RawOnionHopData {
-                               data: hex::decode("fd0100000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff").unwrap(),
-                       },
-                       RawOnionHopData {
-                               data: hex::decode("140303030303030303000000000000000300000003").unwrap(),
-                       },
-                       RawOnionHopData::new(msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0404040404040404,
-                               },
-                               amt_to_forward: 4,
-                               outgoing_cltv_value: 4,
-                       }),
-               );
-
-               let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32])));
-               // Just check the final packet encoding, as it includes all the per-hop vectors in it
-               // anyway...
-               assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a71a060daf367132b378b3a3883c0e2c0e026b8900b2b5cdbc784e1a3bb913f88a9c50f7d61ab590531cf08000178a333a347f8b4072ed056f820f77774345e183a342ec4729f3d84accf515e88adddb85ecc08daba68404bae9a8e8d7178977d7094a1ae549f89338c0777551f874159eb42d3a59fb9285ad4e24883f27de23942ec966611e99bee1cee503455be9e8e642cef6cef7b9864130f692283f8a973d47a8f1c1726b6e59969385975c766e35737c8d76388b64f748ee7943ffb0e2ee45c57a1abc40762ae598723d21bd184e2b338f68ebff47219357bd19cd7e01e2337b806ef4d717888e129e59cd3dc31e6201ccb2fd6d7499836f37a993262468bcb3a4dcd03a22818aca49c6b7b9b8e9e870045631d8e039b066ff86e0d1b7291f71cefa7264c70404a8e538b566c17ccc5feab231401e6c08a01bd5edfc1aa8e3e533b96e82d1f91118d508924b923531929aea889fcdf050597c681185f336b1da63b0939aa2b7c50b21b5eb7b6ad66c81fab98a3cdf73f658149e7e9ced4edde5d38c9b8f92e16f6b4ab13d7fca6a0e4ecc9f9de611a90da6e99c39551094c56e3196f282c5dffd9fc4b2fc12f3bca8e6fe47eb45fbdd3be21a8a8d200797eae3c9a0497132f92410d804977408494dff49dd3d8bce248e0b74fd9e6f0f7102c25ddfa02bd9ad9f746abbfa337ef811d5345a9e16b60de1767b209645ba40bd1f9a5f75bc04feca9b27c5554be4fe83fac2cb83aa447a817bb85ae966c68b420063833fada375e2f515965e687a45699632902672c654d1d18d7bcbf55e8fa57f63f2da449f8e1e606e8722df081e5f193fc4179feb99ad22819afdeef211f7c54afdba92aeef0c00b7bc2b65a4813c01f907a8377585708f2d4c940a25328e585714c8ded0a9a4d7a6de1027c1cb7a0198cd3db68b58c0704dfd0cfbe624e9cd18cc0ae5d96697bb476708b9ee0403d211e64e0d5a7683a7a9a140c02f0ff1c6e67a302941b4052bdea8a63e70a3ad62c5b89c698f1fd3c7685cb49705096cad702d02d93bcb1c27a409f4c9bddec001205ca4a2740f19b50900be81c7e847f1a863deea8d35701f1355cad8db57b1d4eb2ab4e29587734785abfb46ddede71928213d7d089dfdeda052827f459f1688cc0935bd47e7bcec27427c8376dcce7e22699567c0d145f8a7db33f6758815f1f15f9f7a9760dec4f34ae095edda4c64e9735bdd029c4e32c2ee31ba47ec5e6bdb97813d52dbd15b4e0b7a2c7f790ae64104d99f38c127f0a093288fa34144adb16b8968d4fa7656fcec99de8503dd46d3b03620a71c7cd085364abd30dccf7fbda25a1cdc102600149c9af1c97aa0372cd2e1909f28ac5c686f432b310e79528c9b8b9e8f314c1e74621ce6308ad2278b81d460892e0d9dd38b7c76d58be6dfd10ae7583ee1e7ef5b3f6f78dc60af0950df1b00cc55b6d178ba2e476bea0eaeef49323b83f05804159e7aef4eed4cc60dd07be76f067dfd0bcfb0b806b69ba921336a20c43c832d0cab8fa3ddeb29e3bf07b0d98a112eb07802756235a49d44a8b82a950d84e95e01971f0e106ccb337f07384e21620e0ad39e16ed9edca123226cf55ac44f449eeb53e38a7f27d101806e4823e4efcc887414240ee6826c4a5cb1c6443ad36ebf905a435c1d9054e54173911b17b5b40f60b3d9fd5f12eac54ca1e20191f5f18544d5fd3d665e9bcef96fb44b76110aa64d9db4c86c9513cbdad546538e8aec521fbe83ceac5e74a15629f1ed0b870a1d0d1e5680b6d6100d1bd3f3b9043bd35b8919c4088f1949b8be89e4701eb870f8ed64fafa446c78df3ea").unwrap());
-       }
 }
index c22f4611c34438b80f3071de6635b9ae43410402..a65da368709a77aa84147b86dc11e55ae367a558 100644 (file)
@@ -16,7 +16,7 @@ use crate::chain::channelmonitor::{ANTI_REORG_DELAY, ChannelMonitor, LATENCY_GRA
 use crate::chain::transaction::OutPoint;
 use crate::chain::keysinterface::KeysInterface;
 use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
-use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT, ChannelManager, ChannelManagerReadArgs, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure};
+use crate::ln::channelmanager::{self, BREAKDOWN_TIMEOUT, ChannelManager, ChannelManagerReadArgs, MPP_TIMEOUT_TICKS, MIN_CLTV_EXPIRY_DELTA, PaymentId, PaymentSendFailure, IDEMPOTENCY_TIMEOUT_TICKS};
 use crate::ln::msgs;
 use crate::ln::msgs::ChannelMessageHandler;
 use crate::routing::router::{PaymentParameters, get_route};
@@ -53,7 +53,8 @@ fn retry_single_path_payment() {
        send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 
        // Make sure the payment fails on the first hop.
-       let payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       let payment_id = PaymentId(payment_hash.0);
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -138,7 +139,8 @@ fn mpp_retry() {
        route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 
        // Initiate the MPP payment.
-       let payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       let payment_id = PaymentId(payment_hash.0);
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 2); // one monitor per path
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 2);
@@ -222,7 +224,7 @@ fn do_mpp_receive_timeout(send_partial_mpp: bool) {
        route.paths[1][1].short_channel_id = chan_4_update.contents.short_channel_id;
 
        // Initiate the MPP payment.
-       let _ = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 2); // one monitor per path
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 2);
@@ -290,7 +292,7 @@ fn retry_expired_payment() {
        send_payment(&nodes[1], &vec!(&nodes[2])[..], 2_000_000);
 
        // Make sure the payment fails on the first hop.
-       let payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -314,7 +316,7 @@ fn retry_expired_payment() {
        connect_blocks(&nodes[0], 3);
 
        // Retry the payment and make sure it errors as expected.
-       if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, payment_id) {
+       if let Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError { err })) = nodes[0].node.retry_payment(&route, PaymentId(payment_hash.0)) {
                assert!(err.contains("not found"));
        } else {
                panic!("Unexpected error");
@@ -341,7 +343,7 @@ fn no_pending_leak_on_initial_send_failure() {
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
        nodes[1].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 
-       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)),
+       unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)),
                true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Peer for first hop currently disconnected/pending monitor update!"));
 
@@ -365,7 +367,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
@@ -378,7 +380,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        // out and retry.
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
        let (payment_preimage_1, payment_hash_1, _, payment_id_1) = send_along_route(&nodes[0], route.clone(), &[&nodes[1], &nodes[2]], 1_000_000);
-       let payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -423,7 +425,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                        default_config: test_default_channel_config(),
                        keys_manager,
                        fee_estimator: node_cfgs[0].fee_estimator,
@@ -483,6 +485,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &htlc_fulfill_updates.update_fulfill_htlcs[0]);
        check_added_monitors!(nodes[1], 1);
        commitment_signed_dance!(nodes[1], nodes[2], htlc_fulfill_updates.commitment_signed, false);
+       expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 
        if confirm_before_reload {
                let best_block = nodes[0].blocks.lock().unwrap().last().unwrap().clone();
@@ -499,7 +502,6 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        let bs_htlc_claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(bs_htlc_claim_txn.len(), 1);
        check_spends!(bs_htlc_claim_txn[0], as_commitment_tx);
-       expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], None, false, false);
 
        if !confirm_before_reload {
                mine_transaction(&nodes[0], &as_commitment_tx);
@@ -543,7 +545,7 @@ fn do_retry_with_no_persist(confirm_before_reload: bool) {
        }
 
        assert!(nodes[0].node.retry_payment(&new_route, payment_id_1).is_err()); // Shouldn't be allowed to retry a fulfilled payment
-       nodes[0].node.retry_payment(&new_route, payment_id).unwrap();
+       nodes[0].node.retry_payment(&new_route, PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 1);
@@ -574,13 +576,13 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 
        let first_persister: test_utils::TestPersister;
        let first_new_chain_monitor: test_utils::TestChainMonitor;
-       let first_nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let first_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let second_persister: test_utils::TestPersister;
        let second_new_chain_monitor: test_utils::TestChainMonitor;
-       let second_nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let second_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let third_persister: test_utils::TestPersister;
        let third_new_chain_monitor: test_utils::TestChainMonitor;
-       let third_nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let third_nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
@@ -631,7 +633,7 @@ fn do_test_completed_payment_not_retryable_on_reload(use_dust: bool) {
 
                        let mut nodes_0_read = &nodes_0_serialized[..];
                        let (_, nodes_0_deserialized_tmp) = {
-                               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
+                               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
                                        default_config: test_default_channel_config(),
                                        keys_manager,
                                        fee_estimator: node_cfgs[0].fee_estimator,
@@ -799,7 +801,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let (_, _, chan_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
@@ -915,7 +917,7 @@ fn do_test_dup_htlc_onchain_fails_on_reload(persist_manager_post_event: bool, co
        let (_, nodes_0_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>
                        ::read(&mut io::Cursor::new(&chan_manager_serialized.0[..]), ChannelManagerReadArgs {
                                default_config: Default::default(),
                                keys_manager,
@@ -972,7 +974,7 @@ fn test_fulfill_restart_failure() {
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_1_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
@@ -1006,7 +1008,7 @@ fn test_fulfill_restart_failure() {
        let (_, nodes_1_deserialized_tmp) = {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>
                        ::read(&mut io::Cursor::new(&chan_manager_serialized.0[..]), ChannelManagerReadArgs {
                                default_config: Default::default(),
                                keys_manager,
@@ -1060,7 +1062,7 @@ fn get_ldk_payment_preimage() {
                &nodes[0].node.get_our_node_id(), &payment_params, &nodes[0].network_graph.read_only(),
                Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
                amt_msat, TEST_FINAL_CLTV, nodes[0].logger, &scorer, &random_seed_bytes).unwrap();
-       let _payment_id = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        // Make sure to use `get_payment_preimage`
@@ -1253,3 +1255,133 @@ fn onchain_failed_probe_yields_event() {
        }
        assert!(found_probe_failed);
 }
+
+#[test]
+fn claimed_send_payment_idempotent() {
+       // Tests that `send_payment` (and friends) are (reasonably) idempotent.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
+       let (first_payment_preimage, _, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
+
+       macro_rules! check_send_rejected {
+               () => {
+                       // If we try to resend a new payment with a different payment_hash but with the same
+                       // payment_id, it should be rejected.
+                       let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
+                       match send_result {
+                               Err(PaymentSendFailure::ParameterError(APIError::RouteError { err: "Payment already in progress" })) => {},
+                               _ => panic!("Unexpected send result: {:?}", send_result),
+                       }
+
+                       // Further, if we try to send a spontaneous payment with the same payment_id it should
+                       // also be rejected.
+                       let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
+                       match send_result {
+                               Err(PaymentSendFailure::ParameterError(APIError::RouteError { err: "Payment already in progress" })) => {},
+                               _ => panic!("Unexpected send result: {:?}", send_result),
+                       }
+               }
+       }
+
+       check_send_rejected!();
+
+       // Claim the payment backwards, but note that the PaymentSent event is still pending and has
+       // not been seen by the user. At this point, from the user perspective nothing has changed, so
+       // we must remain just as idempotent as we were before.
+       do_claim_payment_along_route(&nodes[0], &[&[&nodes[1]]], false, first_payment_preimage);
+
+       for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
+               nodes[0].node.timer_tick_occurred();
+       }
+
+       check_send_rejected!();
+
+       // Once the user sees and handles the `PaymentSent` event, we expect them to no longer call
+       // `send_payment`, and our idempotency guarantees are off - they should have atomically marked
+       // the payment complete. However, they could have called `send_payment` while the event was
+       // being processed, leading to a race in our idempotency guarantees. Thus, even immediately
+       // after the event is handled a duplicate payment should sitll be rejected.
+       expect_payment_sent!(&nodes[0], first_payment_preimage, Some(0));
+       check_send_rejected!();
+
+       // If relatively little time has passed, a duplicate payment should still fail.
+       nodes[0].node.timer_tick_occurred();
+       check_send_rejected!();
+
+       // However, after some time has passed (at least more than the one timer tick above), a
+       // duplicate payment should go through, as ChannelManager should no longer have any remaining
+       // references to the old payment data.
+       for _ in 0..IDEMPOTENCY_TIMEOUT_TICKS {
+               nodes[0].node.timer_tick_occurred();
+       }
+
+       nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
+       check_added_monitors!(nodes[0], 1);
+       pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
+       claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
+}
+
+#[test]
+fn abandoned_send_payment_idempotent() {
+       // Tests that `send_payment` (and friends) allow duplicate PaymentIds immediately after
+       // abandon_payment.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
+
+       let (route, second_payment_hash, second_payment_preimage, second_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
+       let (_, first_payment_hash, _, payment_id) = send_along_route(&nodes[0], route.clone(), &[&nodes[1]], 100_000);
+
+       macro_rules! check_send_rejected {
+               () => {
+                       // If we try to resend a new payment with a different payment_hash but with the same
+                       // payment_id, it should be rejected.
+                       let send_result = nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id);
+                       match send_result {
+                               Err(PaymentSendFailure::ParameterError(APIError::RouteError { err: "Payment already in progress" })) => {},
+                               _ => panic!("Unexpected send result: {:?}", send_result),
+                       }
+
+                       // Further, if we try to send a spontaneous payment with the same payment_id it should
+                       // also be rejected.
+                       let send_result = nodes[0].node.send_spontaneous_payment(&route, None, payment_id);
+                       match send_result {
+                               Err(PaymentSendFailure::ParameterError(APIError::RouteError { err: "Payment already in progress" })) => {},
+                               _ => panic!("Unexpected send result: {:?}", send_result),
+                       }
+               }
+       }
+
+       check_send_rejected!();
+
+       nodes[1].node.fail_htlc_backwards(&first_payment_hash);
+       expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::FailedPayment { payment_hash: first_payment_hash }]);
+
+       pass_failed_payment_back_no_abandon(&nodes[0], &[&[&nodes[1]]], false, first_payment_hash);
+       check_send_rejected!();
+
+       // Until we abandon the payment, no matter how many timer ticks pass, we still cannot reuse the
+       // PaymentId.
+       for _ in 0..=IDEMPOTENCY_TIMEOUT_TICKS {
+               nodes[0].node.timer_tick_occurred();
+       }
+       check_send_rejected!();
+
+       nodes[0].node.abandon_payment(payment_id);
+       get_event!(nodes[0], Event::PaymentFailed);
+
+       // However, we can reuse the PaymentId immediately after we `abandon_payment`.
+       nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret), payment_id).unwrap();
+       check_added_monitors!(nodes[0], 1);
+       pass_along_route(&nodes[0], &[&[&nodes[1]]], 100_000, second_payment_hash, second_payment_secret);
+       claim_payment(&nodes[0], &[&nodes[1]], second_payment_preimage);
+}
index 228c2491c4793eda09b6fbad299d5ee701b9fceb..351f5142604ef2bb6df0cf528720148febcb59fc 100644 (file)
@@ -21,7 +21,7 @@ use crate::ln::features::{InitFeatures, NodeFeatures};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, LightningError, NetAddress, OnionMessageHandler, RoutingMessageHandler};
 use crate::ln::channelmanager::{SimpleArcChannelManager, SimpleRefChannelManager};
-use crate::util::ser::{MaybeReadableArgs, VecWriter, Writeable, Writer};
+use crate::util::ser::{VecWriter, Writeable, Writer};
 use crate::ln::peer_channel_encryptor::{PeerChannelEncryptor,NextNoiseStep};
 use crate::ln::wire;
 use crate::ln::wire::Encode;
@@ -97,13 +97,11 @@ impl OnionMessageHandler for IgnoringMessageHandler {
 }
 impl CustomOnionMessageHandler for IgnoringMessageHandler {
        type CustomMessage = Infallible;
-       fn handle_custom_message(&self, _msg: Self::CustomMessage) {
+       fn handle_custom_message(&self, _msg: Infallible) {
                // Since we always return `None` in the read the handle method should never be called.
                unreachable!();
        }
-}
-impl MaybeReadableArgs<u64> for Infallible {
-       fn read<R: io::Read>(_buffer: &mut R, _msg_type: u64) -> Result<Option<Self>, msgs::DecodeError> where Self: Sized {
+       fn read_custom_message<R: io::Read>(&self, _msg_type: u64, _buffer: &mut R) -> Result<Option<Infallible>, msgs::DecodeError> where Self: Sized {
                Ok(None)
        }
 }
index cea6f37c85d004cdf52d7f993f11fb4bd88bd835..6e2cf2583c43567e41fd5e22697729de9f6c2431 100644 (file)
@@ -14,7 +14,7 @@
 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
 use crate::chain::channelmonitor::ChannelMonitor;
 use crate::chain::keysinterface::{Recipient, KeysInterface};
-use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, MIN_CLTV_EXPIRY_DELTA};
+use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, MIN_CLTV_EXPIRY_DELTA, PaymentId};
 use crate::routing::gossip::RoutingFees;
 use crate::routing::router::{PaymentParameters, RouteHint, RouteHintHop};
 use crate::ln::features::ChannelTypeFeatures;
@@ -51,7 +51,7 @@ fn test_priv_forwarding_rejection() {
        let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(no_announce_cfg), None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_1_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 
        let chan_id_1 = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 500_000_000, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
@@ -77,7 +77,7 @@ fn test_priv_forwarding_rejection() {
                .with_route_hints(last_hops);
        let (route, our_payment_hash, our_payment_preimage, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 10_000, TEST_FINAL_CLTV);
 
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let payment_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
@@ -124,7 +124,7 @@ fn test_priv_forwarding_rejection() {
                let mut channel_monitors = HashMap::new();
                channel_monitors.insert(monitor_a.get_funding_txo().0, &mut monitor_a);
                channel_monitors.insert(monitor_b.get_funding_txo().0, &mut monitor_b);
-               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
                        default_config: no_announce_cfg,
                        keys_manager,
                        fee_estimator: node_cfgs[1].fee_estimator,
@@ -162,7 +162,7 @@ fn test_priv_forwarding_rejection() {
        get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[2].node.get_our_node_id());
        get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[1].node.get_our_node_id());
 
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 10_000, our_payment_hash, our_payment_secret);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], our_payment_preimage);
@@ -202,6 +202,8 @@ fn do_test_1_conf_open(connect_style: ConnectStyle) {
        } else { panic!("Unexpected event"); }
 
        nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
+       expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
+       expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
        let bs_msg_events = nodes[1].node.get_and_clear_pending_msg_events();
        assert_eq!(bs_msg_events.len(), 1);
        if let MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } = bs_msg_events[0] {
@@ -280,7 +282,7 @@ fn test_routed_scid_alias() {
                .with_route_hints(hop_hints);
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 100_000, 42);
        assert_eq!(route.paths[0][1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 100_000, payment_hash, payment_secret);
@@ -407,8 +409,10 @@ fn test_inbound_scid_privacy() {
        connect_blocks(&nodes[2], CHAN_CONFIRM_DEPTH - 1);
        let bs_channel_ready = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReady, nodes[2].node.get_our_node_id());
        nodes[1].node.handle_channel_ready(&nodes[2].node.get_our_node_id(), &get_event_msg!(nodes[2], MessageSendEvent::SendChannelReady, nodes[1].node.get_our_node_id()));
+       expect_channel_ready_event(&nodes[1], &nodes[2].node.get_our_node_id());
        let bs_update = get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[2].node.get_our_node_id());
        nodes[2].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
+       expect_channel_ready_event(&nodes[2], &nodes[1].node.get_our_node_id());
        let cs_update = get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[1].node.get_our_node_id());
 
        nodes[1].node.handle_channel_update(&nodes[2].node.get_our_node_id(), &cs_update);
@@ -433,7 +437,7 @@ fn test_inbound_scid_privacy() {
                .with_route_hints(hop_hints.clone());
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 100_000, 42);
        assert_eq!(route.paths[0][1].short_channel_id, last_hop[0].inbound_scid_alias.unwrap());
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 100_000, payment_hash, payment_secret);
@@ -448,7 +452,7 @@ fn test_inbound_scid_privacy() {
                .with_route_hints(hop_hints);
        let (route_2, payment_hash_2, _, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params_2, 100_000, 42);
        assert_eq!(route_2.paths[0][1].short_channel_id, last_hop[0].short_channel_id.unwrap());
-       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let payment_event = SendEvent::from_node(&nodes[0]);
@@ -503,7 +507,7 @@ fn test_scid_alias_returned() {
        route.paths[0][1].fee_msat = 10_000_000; // Overshoot the last channel's value
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
@@ -547,7 +551,7 @@ fn test_scid_alias_returned() {
        route.paths[0][0].fee_msat = 0; // But set fee paid to the middle hop to 0
 
        // Route the HTLC through to the destination.
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
@@ -672,6 +676,9 @@ fn test_0conf_channel_with_async_monitor() {
                }
                _ => panic!("Unexpected event"),
        }
+       expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
+       expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
+
        let bs_channel_update = get_event_msg!(nodes[1], MessageSendEvent::SendChannelUpdate, nodes[0].node.get_our_node_id());
 
        let as_channel_update = match &as_locked_update[1] {
@@ -697,7 +704,7 @@ fn test_0conf_channel_with_async_monitor() {
        // failure before we've ever confirmed the funding transaction. This previously caused a panic.
        let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 
-       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
        let as_send = SendEvent::from_node(&nodes[0]);
index 9d02009f74e7ce25378b92f320a8910f95fc8b26..a445f420a9c96011238a28a0eff4c7e0e8fe4426 100644 (file)
@@ -267,22 +267,28 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
        let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
        let persister: test_utils::TestPersister;
        let new_chain_monitor: test_utils::TestChainMonitor;
-       let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
        let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
        *nodes[0].connect_style.borrow_mut() = connect_style;
 
+       let chan_conf_height = core::cmp::max(nodes[0].best_block_info().1 + 1, nodes[1].best_block_info().1 + 1);
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let channel_state = nodes[0].node.channel_state.lock().unwrap();
        assert_eq!(channel_state.by_id.len(), 1);
-       assert_eq!(channel_state.short_to_chan_info.len(), 2);
+       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 2);
        mem::drop(channel_state);
 
        if !reorg_after_reload {
                if use_funding_unconfirmed {
                        let relevant_txids = nodes[0].node.get_relevant_txids();
-                       assert_eq!(&relevant_txids[..], &[chan.3.txid()]);
-                       nodes[0].node.transaction_unconfirmed(&relevant_txids[0]);
+                       assert_eq!(relevant_txids.len(), 1);
+                       let block_hash_opt = relevant_txids[0].1;
+                       let expected_hash = nodes[0].get_block_header(chan_conf_height).block_hash();
+                       assert_eq!(block_hash_opt, Some(expected_hash));
+                       let txid = relevant_txids[0].0;
+                       assert_eq!(txid, chan.3.txid());
+                       nodes[0].node.transaction_unconfirmed(&txid);
                } else if connect_style == ConnectStyle::FullBlockViaListen {
                        disconnect_blocks(&nodes[0], CHAN_CONFIRM_DEPTH - 1);
                        assert_eq!(nodes[0].node.list_usable_channels().len(), 1);
@@ -290,12 +296,16 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                } else {
                        disconnect_all_blocks(&nodes[0]);
                }
+
+               let relevant_txids = nodes[0].node.get_relevant_txids();
+               assert_eq!(relevant_txids.len(), 0);
+
                handle_announce_close_broadcast_events(&nodes, 0, 1, true, "Channel closed because of an exception: Funding transaction was un-confirmed. Locked at 6 confs, now have 0 confs.");
                check_added_monitors!(nodes[1], 1);
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
-                       assert_eq!(channel_state.short_to_chan_info.len(), 0);
+                       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 0);
                }
        }
 
@@ -321,7 +331,7 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                nodes_0_deserialized = {
                        let mut channel_monitors = HashMap::new();
                        channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
-                       <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster,
+                       <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster,
                          &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(
                                &mut nodes_0_read, ChannelManagerReadArgs {
                                        default_config: *nodes[0].node.get_current_default_configuration(),
@@ -350,8 +360,13 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
        if reorg_after_reload {
                if use_funding_unconfirmed {
                        let relevant_txids = nodes[0].node.get_relevant_txids();
-                       assert_eq!(&relevant_txids[..], &[chan.3.txid()]);
-                       nodes[0].node.transaction_unconfirmed(&relevant_txids[0]);
+                       assert_eq!(relevant_txids.len(), 1);
+                       let block_hash_opt = relevant_txids[0].1;
+                       let expected_hash = nodes[0].get_block_header(chan_conf_height).block_hash();
+                       assert_eq!(block_hash_opt, Some(expected_hash));
+                       let txid = relevant_txids[0].0;
+                       assert_eq!(txid, chan.3.txid());
+                       nodes[0].node.transaction_unconfirmed(&txid);
                } else if connect_style == ConnectStyle::FullBlockViaListen {
                        disconnect_blocks(&nodes[0], CHAN_CONFIRM_DEPTH - 1);
                        assert_eq!(nodes[0].node.list_channels().len(), 1);
@@ -359,12 +374,16 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
                } else {
                        disconnect_all_blocks(&nodes[0]);
                }
+
+               let relevant_txids = nodes[0].node.get_relevant_txids();
+               assert_eq!(relevant_txids.len(), 0);
+
                handle_announce_close_broadcast_events(&nodes, 0, 1, true, "Channel closed because of an exception: Funding transaction was un-confirmed. Locked at 6 confs, now have 0 confs.");
                check_added_monitors!(nodes[1], 1);
                {
                        let channel_state = nodes[0].node.channel_state.lock().unwrap();
                        assert_eq!(channel_state.by_id.len(), 0);
-                       assert_eq!(channel_state.short_to_chan_info.len(), 0);
+                       assert_eq!(nodes[0].node.short_to_chan_info.read().unwrap().len(), 0);
                }
        }
        // With expect_channel_force_closed set the TestChainMonitor will enforce that the next update
index c093fe2ae219d900c238cb5ff26d5cd2e4ae4266..48b452cdd9fe75b0b84031a0033ff9706001e0e8 100644 (file)
@@ -11,7 +11,7 @@
 
 use crate::chain::keysinterface::KeysInterface;
 use crate::chain::transaction::OutPoint;
-use crate::ln::channelmanager::{self, PaymentSendFailure};
+use crate::ln::channelmanager::{self, PaymentSendFailure, PaymentId};
 use crate::routing::router::{PaymentParameters, get_route};
 use crate::ln::msgs;
 use crate::ln::msgs::{ChannelMessageHandler, ErrorAction};
@@ -97,8 +97,8 @@ fn updates_shutdown_wait() {
        let route_1 = get_route(&nodes[0].node.get_our_node_id(), &payment_params_1, &nodes[0].network_graph.read_only(), None, 100000, TEST_FINAL_CLTV, &logger, &scorer, &random_seed_bytes).unwrap();
        let payment_params_2 = PaymentParameters::from_node_id(nodes[0].node.get_our_node_id()).with_features(channelmanager::provided_invoice_features());
        let route_2 = get_route(&nodes[1].node.get_our_node_id(), &payment_params_2, &nodes[1].network_graph.read_only(), None, 100000, TEST_FINAL_CLTV, &logger, &scorer, &random_seed_bytes).unwrap();
-       unwrap_send_err!(nodes[0].node.send_payment(&route_1, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable {..}, {});
-       unwrap_send_err!(nodes[1].node.send_payment(&route_2, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[0].node.send_payment(&route_1, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable {..}, {});
+       unwrap_send_err!(nodes[1].node.send_payment(&route_2, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)), true, APIError::ChannelUnavailable {..}, {});
 
        nodes[2].node.claim_funds(payment_preimage_0);
        check_added_monitors!(nodes[2], 1);
@@ -158,7 +158,7 @@ fn htlc_fail_async_shutdown() {
        let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 
        let (route, our_payment_hash, _, our_payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 100000);
-       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret), PaymentId(our_payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        assert_eq!(updates.update_add_htlcs.len(), 1);
diff --git a/lightning/src/offers/mod.rs b/lightning/src/offers/mod.rs
new file mode 100644 (file)
index 0000000..2f961a0
--- /dev/null
@@ -0,0 +1,15 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Implementation of Lightning Offers
+//! ([BOLT 12](https://github.com/lightning/bolts/blob/master/12-offer-encoding.md)).
+//!
+//! Offers are a flexible protocol for Lightning payments.
+
+pub mod offer;
diff --git a/lightning/src/offers/offer.rs b/lightning/src/offers/offer.rs
new file mode 100644 (file)
index 0000000..bf569fb
--- /dev/null
@@ -0,0 +1,709 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Data structures and encoding for `offer` messages.
+//!
+//! An [`Offer`] represents an "offer to be paid." It is typically constructed by a merchant and
+//! published as a QR code to be scanned by a customer. The customer uses the offer to request an
+//! invoice from the merchant to be paid.
+//!
+//! ```ignore
+//! extern crate bitcoin;
+//! extern crate core;
+//! extern crate lightning;
+//!
+//! use core::num::NonZeroU64;
+//! use core::time::Duration;
+//!
+//! use bitcoin::secp256k1::{KeyPair, PublicKey, Secp256k1, SecretKey};
+//! use lightning::offers::offer::{OfferBuilder, Quantity};
+//!
+//! # use bitcoin::secp256k1;
+//! # use lightning::onion_message::BlindedPath;
+//! # #[cfg(feature = "std")]
+//! # use std::time::SystemTime;
+//! #
+//! # fn create_blinded_path() -> BlindedPath { unimplemented!() }
+//! # fn create_another_blinded_path() -> BlindedPath { unimplemented!() }
+//! #
+//! # #[cfg(feature = "std")]
+//! # fn build() -> Result<(), secp256k1::Error> {
+//! let secp_ctx = Secp256k1::new();
+//! let keys = KeyPair::from_secret_key(&secp_ctx, &SecretKey::from_slice(&[42; 32])?);
+//! let pubkey = PublicKey::from(keys);
+//!
+//! let expiration = SystemTime::now() + Duration::from_secs(24 * 60 * 60);
+//! let offer = OfferBuilder::new("coffee, large".to_string(), pubkey)
+//!     .amount_msats(20_000)
+//!     .supported_quantity(Quantity::Unbounded)
+//!     .absolute_expiry(expiration.duration_since(SystemTime::UNIX_EPOCH).unwrap())
+//!     .issuer("Foo Bar".to_string())
+//!     .path(create_blinded_path())
+//!     .path(create_another_blinded_path())
+//!     .build()
+//!     .unwrap();
+//! # Ok(())
+//! # }
+//! ```
+
+use bitcoin::blockdata::constants::ChainHash;
+use bitcoin::network::constants::Network;
+use bitcoin::secp256k1::PublicKey;
+use core::num::NonZeroU64;
+use core::time::Duration;
+use crate::io;
+use crate::ln::features::OfferFeatures;
+use crate::ln::msgs::MAX_VALUE_MSAT;
+use crate::onion_message::BlindedPath;
+use crate::util::ser::{HighZeroBytesDroppedBigSize, WithoutLength, Writeable, Writer};
+use crate::util::string::PrintableString;
+
+use crate::prelude::*;
+
+#[cfg(feature = "std")]
+use std::time::SystemTime;
+
+/// Builds an [`Offer`] for the "offer to be paid" flow.
+///
+/// See [module-level documentation] for usage.
+///
+/// [module-level documentation]: self
+pub struct OfferBuilder {
+       offer: OfferContents,
+}
+
+impl OfferBuilder {
+       /// Creates a new builder for an offer setting the [`Offer::description`] and using the
+       /// [`Offer::signing_pubkey`] for signing invoices. The associated secret key must be remembered
+       /// while the offer is valid.
+       ///
+       /// Use a different pubkey per offer to avoid correlating offers.
+       pub fn new(description: String, signing_pubkey: PublicKey) -> Self {
+               let offer = OfferContents {
+                       chains: None, metadata: None, amount: None, description,
+                       features: OfferFeatures::empty(), absolute_expiry: None, issuer: None, paths: None,
+                       supported_quantity: Quantity::one(), signing_pubkey: Some(signing_pubkey),
+               };
+               OfferBuilder { offer }
+       }
+
+       /// Adds the chain hash of the given [`Network`] to [`Offer::chains`]. If not called,
+       /// the chain hash of [`Network::Bitcoin`] is assumed to be the only one supported.
+       ///
+       /// See [`Offer::chains`] on how this relates to the payment currency.
+       ///
+       /// Successive calls to this method will add another chain hash.
+       pub fn chain(mut self, network: Network) -> Self {
+               let chains = self.offer.chains.get_or_insert_with(Vec::new);
+               let chain = ChainHash::using_genesis_block(network);
+               if !chains.contains(&chain) {
+                       chains.push(chain);
+               }
+
+               self
+       }
+
+       /// Sets the [`Offer::metadata`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn metadata(mut self, metadata: Vec<u8>) -> Self {
+               self.offer.metadata = Some(metadata);
+               self
+       }
+
+       /// Sets the [`Offer::amount`] as an [`Amount::Bitcoin`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn amount_msats(mut self, amount_msats: u64) -> Self {
+               self.amount(Amount::Bitcoin { amount_msats })
+       }
+
+       /// Sets the [`Offer::amount`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       fn amount(mut self, amount: Amount) -> Self {
+               self.offer.amount = Some(amount);
+               self
+       }
+
+       /// Sets the [`Offer::features`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       #[cfg(test)]
+       pub fn features(mut self, features: OfferFeatures) -> Self {
+               self.offer.features = features;
+               self
+       }
+
+       /// Sets the [`Offer::absolute_expiry`] as seconds since the Unix epoch. Any expiry that has
+       /// already passed is valid and can be checked for using [`Offer::is_expired`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn absolute_expiry(mut self, absolute_expiry: Duration) -> Self {
+               self.offer.absolute_expiry = Some(absolute_expiry);
+               self
+       }
+
+       /// Sets the [`Offer::issuer`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn issuer(mut self, issuer: String) -> Self {
+               self.offer.issuer = Some(issuer);
+               self
+       }
+
+       /// Adds a blinded path to [`Offer::paths`]. Must include at least one path if only connected by
+       /// private channels or if [`Offer::signing_pubkey`] is not a public node id.
+       ///
+       /// Successive calls to this method will add another blinded path. Caller is responsible for not
+       /// adding duplicate paths.
+       pub fn path(mut self, path: BlindedPath) -> Self {
+               self.offer.paths.get_or_insert_with(Vec::new).push(path);
+               self
+       }
+
+       /// Sets the quantity of items for [`Offer::supported_quantity`].
+       ///
+       /// Successive calls to this method will override the previous setting.
+       pub fn supported_quantity(mut self, quantity: Quantity) -> Self {
+               self.offer.supported_quantity = quantity;
+               self
+       }
+
+       /// Builds an [`Offer`] from the builder's settings.
+       pub fn build(mut self) -> Result<Offer, ()> {
+               match self.offer.amount {
+                       Some(Amount::Bitcoin { amount_msats }) => {
+                               if amount_msats > MAX_VALUE_MSAT {
+                                       return Err(());
+                               }
+                       },
+                       Some(Amount::Currency { .. }) => unreachable!(),
+                       None => {},
+               }
+
+               if let Some(chains) = &self.offer.chains {
+                       if chains.len() == 1 && chains[0] == self.offer.implied_chain() {
+                               self.offer.chains = None;
+                       }
+               }
+
+               let mut bytes = Vec::new();
+               self.offer.write(&mut bytes).unwrap();
+
+               Ok(Offer {
+                       bytes,
+                       contents: self.offer,
+               })
+       }
+}
+
+/// An `Offer` is a potentially long-lived proposal for payment of a good or service.
+///
+/// An offer is a precursor to an `InvoiceRequest`. A merchant publishes an offer from which a
+/// customer may request an `Invoice` for a specific quantity and using an amount sufficient to
+/// cover that quantity (i.e., at least `quantity * amount`). See [`Offer::amount`].
+///
+/// Offers may be denominated in currency other than bitcoin but are ultimately paid using the
+/// latter.
+///
+/// Through the use of [`BlindedPath`]s, offers provide recipient privacy.
+#[derive(Clone, Debug)]
+pub struct Offer {
+       // The serialized offer. Needed when creating an `InvoiceRequest` if the offer contains unknown
+       // fields.
+       bytes: Vec<u8>,
+       contents: OfferContents,
+}
+
+/// The contents of an [`Offer`], which may be shared with an `InvoiceRequest` or an `Invoice`.
+#[derive(Clone, Debug)]
+pub(crate) struct OfferContents {
+       chains: Option<Vec<ChainHash>>,
+       metadata: Option<Vec<u8>>,
+       amount: Option<Amount>,
+       description: String,
+       features: OfferFeatures,
+       absolute_expiry: Option<Duration>,
+       issuer: Option<String>,
+       paths: Option<Vec<BlindedPath>>,
+       supported_quantity: Quantity,
+       signing_pubkey: Option<PublicKey>,
+}
+
+impl Offer {
+       // TODO: Return a slice once ChainHash has constants.
+       // - https://github.com/rust-bitcoin/rust-bitcoin/pull/1283
+       // - https://github.com/rust-bitcoin/rust-bitcoin/pull/1286
+       /// The chains that may be used when paying a requested invoice (e.g., bitcoin mainnet).
+       /// Payments must be denominated in units of the minimal lightning-payable unit (e.g., msats)
+       /// for the selected chain.
+       pub fn chains(&self) -> Vec<ChainHash> {
+               self.contents.chains
+                       .as_ref()
+                       .cloned()
+                       .unwrap_or_else(|| vec![self.contents.implied_chain()])
+       }
+
+       // TODO: Link to corresponding method in `InvoiceRequest`.
+       /// Opaque bytes set by the originator. Useful for authentication and validating fields since it
+       /// is reflected in `invoice_request` messages along with all the other fields from the `offer`.
+       pub fn metadata(&self) -> Option<&Vec<u8>> {
+               self.contents.metadata.as_ref()
+       }
+
+       /// The minimum amount required for a successful payment of a single item.
+       pub fn amount(&self) -> Option<&Amount> {
+               self.contents.amount.as_ref()
+       }
+
+       /// A complete description of the purpose of the payment. Intended to be displayed to the user
+       /// but with the caveat that it has not been verified in any way.
+       pub fn description(&self) -> PrintableString {
+               PrintableString(&self.contents.description)
+       }
+
+       /// Features pertaining to the offer.
+       pub fn features(&self) -> &OfferFeatures {
+               &self.contents.features
+       }
+
+       /// Duration since the Unix epoch when an invoice should no longer be requested.
+       ///
+       /// If `None`, the offer does not expire.
+       pub fn absolute_expiry(&self) -> Option<Duration> {
+               self.contents.absolute_expiry
+       }
+
+       /// Whether the offer has expired.
+       #[cfg(feature = "std")]
+       pub fn is_expired(&self) -> bool {
+               match self.absolute_expiry() {
+                       Some(seconds_from_epoch) => match SystemTime::UNIX_EPOCH.elapsed() {
+                               Ok(elapsed) => elapsed > seconds_from_epoch,
+                               Err(_) => false,
+                       },
+                       None => false,
+               }
+       }
+
+       /// The issuer of the offer, possibly beginning with `user@domain` or `domain`. Intended to be
+       /// displayed to the user but with the caveat that it has not been verified in any way.
+       pub fn issuer(&self) -> Option<PrintableString> {
+               self.contents.issuer.as_ref().map(|issuer| PrintableString(issuer.as_str()))
+       }
+
+       /// Paths to the recipient originating from publicly reachable nodes. Blinded paths provide
+       /// recipient privacy by obfuscating its node id.
+       pub fn paths(&self) -> &[BlindedPath] {
+               self.contents.paths.as_ref().map(|paths| paths.as_slice()).unwrap_or(&[])
+       }
+
+       /// The quantity of items supported.
+       pub fn supported_quantity(&self) -> Quantity {
+               self.contents.supported_quantity()
+       }
+
+       /// The public key used by the recipient to sign invoices.
+       pub fn signing_pubkey(&self) -> PublicKey {
+               self.contents.signing_pubkey.unwrap()
+       }
+
+       #[cfg(test)]
+       fn as_tlv_stream(&self) -> OfferTlvStreamRef {
+               self.contents.as_tlv_stream()
+       }
+}
+
+impl OfferContents {
+       pub fn implied_chain(&self) -> ChainHash {
+               ChainHash::using_genesis_block(Network::Bitcoin)
+       }
+
+       pub fn supported_quantity(&self) -> Quantity {
+               self.supported_quantity
+       }
+
+       fn as_tlv_stream(&self) -> OfferTlvStreamRef {
+               let (currency, amount) = match &self.amount {
+                       None => (None, None),
+                       Some(Amount::Bitcoin { amount_msats }) => (None, Some(*amount_msats)),
+                       Some(Amount::Currency { iso4217_code, amount }) => (
+                               Some(iso4217_code), Some(*amount)
+                       ),
+               };
+
+               let features = {
+                       if self.features == OfferFeatures::empty() { None } else { Some(&self.features) }
+               };
+
+               OfferTlvStreamRef {
+                       chains: self.chains.as_ref(),
+                       metadata: self.metadata.as_ref(),
+                       currency,
+                       amount,
+                       description: Some(&self.description),
+                       features,
+                       absolute_expiry: self.absolute_expiry.map(|duration| duration.as_secs()),
+                       paths: self.paths.as_ref(),
+                       issuer: self.issuer.as_ref(),
+                       quantity_max: self.supported_quantity.to_tlv_record(),
+                       node_id: self.signing_pubkey.as_ref(),
+               }
+       }
+}
+
+impl Writeable for OfferContents {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               self.as_tlv_stream().write(writer)
+       }
+}
+
+/// The minimum amount required for an item in an [`Offer`], denominated in either bitcoin or
+/// another currency.
+#[derive(Clone, Debug, PartialEq)]
+pub enum Amount {
+       /// An amount of bitcoin.
+       Bitcoin {
+               /// The amount in millisatoshi.
+               amount_msats: u64,
+       },
+       /// An amount of currency specified using ISO 4712.
+       Currency {
+               /// The currency that the amount is denominated in.
+               iso4217_code: CurrencyCode,
+               /// The amount in the currency unit adjusted by the ISO 4712 exponent (e.g., USD cents).
+               amount: u64,
+       },
+}
+
+/// An ISO 4712 three-letter currency code (e.g., USD).
+pub type CurrencyCode = [u8; 3];
+
+/// Quantity of items supported by an [`Offer`].
+#[derive(Clone, Copy, Debug, PartialEq)]
+pub enum Quantity {
+       /// Up to a specific number of items (inclusive).
+       Bounded(NonZeroU64),
+       /// One or more items.
+       Unbounded,
+}
+
+impl Quantity {
+       fn one() -> Self {
+               Quantity::Bounded(NonZeroU64::new(1).unwrap())
+       }
+
+       fn to_tlv_record(&self) -> Option<u64> {
+               match self {
+                       Quantity::Bounded(n) => {
+                               let n = n.get();
+                               if n == 1 { None } else { Some(n) }
+                       },
+                       Quantity::Unbounded => Some(0),
+               }
+       }
+}
+
+tlv_stream!(OfferTlvStream, OfferTlvStreamRef, {
+       (2, chains: (Vec<ChainHash>, WithoutLength)),
+       (4, metadata: (Vec<u8>, WithoutLength)),
+       (6, currency: CurrencyCode),
+       (8, amount: (u64, HighZeroBytesDroppedBigSize)),
+       (10, description: (String, WithoutLength)),
+       (12, features: OfferFeatures),
+       (14, absolute_expiry: (u64, HighZeroBytesDroppedBigSize)),
+       (16, paths: (Vec<BlindedPath>, WithoutLength)),
+       (18, issuer: (String, WithoutLength)),
+       (20, quantity_max: (u64, HighZeroBytesDroppedBigSize)),
+       (22, node_id: PublicKey),
+});
+
+#[cfg(test)]
+mod tests {
+       use super::{Amount, OfferBuilder, Quantity};
+
+       use bitcoin::blockdata::constants::ChainHash;
+       use bitcoin::network::constants::Network;
+       use bitcoin::secp256k1::{PublicKey, Secp256k1, SecretKey};
+       use core::num::NonZeroU64;
+       use core::time::Duration;
+       use crate::ln::features::OfferFeatures;
+       use crate::ln::msgs::MAX_VALUE_MSAT;
+       use crate::onion_message::{BlindedHop, BlindedPath};
+       use crate::util::ser::Writeable;
+       use crate::util::string::PrintableString;
+
+       fn pubkey(byte: u8) -> PublicKey {
+               let secp_ctx = Secp256k1::new();
+               PublicKey::from_secret_key(&secp_ctx, &privkey(byte))
+       }
+
+       fn privkey(byte: u8) -> SecretKey {
+               SecretKey::from_slice(&[byte; 32]).unwrap()
+       }
+
+       #[test]
+       fn builds_offer_with_defaults() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42)).build().unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               let mut buffer = Vec::new();
+               offer.contents.write(&mut buffer).unwrap();
+
+               assert_eq!(offer.bytes, buffer.as_slice());
+               assert_eq!(offer.chains(), vec![ChainHash::using_genesis_block(Network::Bitcoin)]);
+               assert_eq!(offer.metadata(), None);
+               assert_eq!(offer.amount(), None);
+               assert_eq!(offer.description(), PrintableString("foo"));
+               assert_eq!(offer.features(), &OfferFeatures::empty());
+               assert_eq!(offer.absolute_expiry(), None);
+               #[cfg(feature = "std")]
+               assert!(!offer.is_expired());
+               assert_eq!(offer.paths(), &[]);
+               assert_eq!(offer.issuer(), None);
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(offer.signing_pubkey(), pubkey(42));
+
+               assert_eq!(tlv_stream.chains, None);
+               assert_eq!(tlv_stream.metadata, None);
+               assert_eq!(tlv_stream.currency, None);
+               assert_eq!(tlv_stream.amount, None);
+               assert_eq!(tlv_stream.description, Some(&String::from("foo")));
+               assert_eq!(tlv_stream.features, None);
+               assert_eq!(tlv_stream.absolute_expiry, None);
+               assert_eq!(tlv_stream.paths, None);
+               assert_eq!(tlv_stream.issuer, None);
+               assert_eq!(tlv_stream.quantity_max, None);
+               assert_eq!(tlv_stream.node_id, Some(&pubkey(42)));
+       }
+
+       #[test]
+       fn builds_offer_with_chains() {
+               let mainnet = ChainHash::using_genesis_block(Network::Bitcoin);
+               let testnet = ChainHash::using_genesis_block(Network::Testnet);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Bitcoin)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![mainnet]);
+               assert_eq!(offer.as_tlv_stream().chains, None);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![testnet]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Testnet)
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![testnet]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .chain(Network::Bitcoin)
+                       .chain(Network::Testnet)
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.chains(), vec![mainnet, testnet]);
+               assert_eq!(offer.as_tlv_stream().chains, Some(&vec![mainnet, testnet]));
+       }
+
+       #[test]
+       fn builds_offer_with_metadata() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .metadata(vec![42; 32])
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.metadata(), Some(&vec![42; 32]));
+               assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![42; 32]));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .metadata(vec![42; 32])
+                       .metadata(vec![43; 32])
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.metadata(), Some(&vec![43; 32]));
+               assert_eq!(offer.as_tlv_stream().metadata, Some(&vec![43; 32]));
+       }
+
+       #[test]
+       fn builds_offer_with_amount() {
+               let bitcoin_amount = Amount::Bitcoin { amount_msats: 1000 };
+               let currency_amount = Amount::Currency { iso4217_code: *b"USD", amount: 10 };
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount_msats(1000)
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.amount(), Some(&bitcoin_amount));
+               assert_eq!(tlv_stream.amount, Some(1000));
+               assert_eq!(tlv_stream.currency, None);
+
+               let builder = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount(currency_amount.clone());
+               let tlv_stream = builder.offer.as_tlv_stream();
+               assert_eq!(builder.offer.amount, Some(currency_amount.clone()));
+               assert_eq!(tlv_stream.amount, Some(10));
+               assert_eq!(tlv_stream.currency, Some(b"USD"));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .amount(currency_amount.clone())
+                       .amount(bitcoin_amount.clone())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(tlv_stream.amount, Some(1000));
+               assert_eq!(tlv_stream.currency, None);
+
+               let invalid_amount = Amount::Bitcoin { amount_msats: MAX_VALUE_MSAT + 1 };
+               match OfferBuilder::new("foo".into(), pubkey(42)).amount(invalid_amount).build() {
+                       Ok(_) => panic!("expected error"),
+                       Err(e) => assert_eq!(e, ()),
+               }
+       }
+
+       #[test]
+       fn builds_offer_with_features() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .features(OfferFeatures::unknown())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.features(), &OfferFeatures::unknown());
+               assert_eq!(offer.as_tlv_stream().features, Some(&OfferFeatures::unknown()));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .features(OfferFeatures::unknown())
+                       .features(OfferFeatures::empty())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.features(), &OfferFeatures::empty());
+               assert_eq!(offer.as_tlv_stream().features, None);
+       }
+
+       #[test]
+       fn builds_offer_with_absolute_expiry() {
+               let future_expiry = Duration::from_secs(u64::max_value());
+               let past_expiry = Duration::from_secs(0);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .absolute_expiry(future_expiry)
+                       .build()
+                       .unwrap();
+               #[cfg(feature = "std")]
+               assert!(!offer.is_expired());
+               assert_eq!(offer.absolute_expiry(), Some(future_expiry));
+               assert_eq!(offer.as_tlv_stream().absolute_expiry, Some(future_expiry.as_secs()));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .absolute_expiry(future_expiry)
+                       .absolute_expiry(past_expiry)
+                       .build()
+                       .unwrap();
+               #[cfg(feature = "std")]
+               assert!(offer.is_expired());
+               assert_eq!(offer.absolute_expiry(), Some(past_expiry));
+               assert_eq!(offer.as_tlv_stream().absolute_expiry, Some(past_expiry.as_secs()));
+       }
+
+       #[test]
+       fn builds_offer_with_paths() {
+               let paths = vec![
+                       BlindedPath {
+                               introduction_node_id: pubkey(40),
+                               blinding_point: pubkey(41),
+                               blinded_hops: vec![
+                                       BlindedHop { blinded_node_id: pubkey(43), encrypted_payload: vec![0; 43] },
+                                       BlindedHop { blinded_node_id: pubkey(44), encrypted_payload: vec![0; 44] },
+                               ],
+                       },
+                       BlindedPath {
+                               introduction_node_id: pubkey(40),
+                               blinding_point: pubkey(41),
+                               blinded_hops: vec![
+                                       BlindedHop { blinded_node_id: pubkey(45), encrypted_payload: vec![0; 45] },
+                                       BlindedHop { blinded_node_id: pubkey(46), encrypted_payload: vec![0; 46] },
+                               ],
+                       },
+               ];
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .path(paths[0].clone())
+                       .path(paths[1].clone())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.paths(), paths.as_slice());
+               assert_eq!(offer.signing_pubkey(), pubkey(42));
+               assert_ne!(pubkey(42), pubkey(44));
+               assert_eq!(tlv_stream.paths, Some(&paths));
+               assert_eq!(tlv_stream.node_id, Some(&pubkey(42)));
+       }
+
+       #[test]
+       fn builds_offer_with_issuer() {
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .issuer("bar".into())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.issuer(), Some(PrintableString("bar")));
+               assert_eq!(offer.as_tlv_stream().issuer, Some(&String::from("bar")));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .issuer("bar".into())
+                       .issuer("baz".into())
+                       .build()
+                       .unwrap();
+               assert_eq!(offer.issuer(), Some(PrintableString("baz")));
+               assert_eq!(offer.as_tlv_stream().issuer, Some(&String::from("baz")));
+       }
+
+       #[test]
+       fn builds_offer_with_supported_quantity() {
+               let ten = NonZeroU64::new(10).unwrap();
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::one())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(tlv_stream.quantity_max, None);
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Unbounded)
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::Unbounded);
+               assert_eq!(tlv_stream.quantity_max, Some(0));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Bounded(ten))
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::Bounded(ten));
+               assert_eq!(tlv_stream.quantity_max, Some(10));
+
+               let offer = OfferBuilder::new("foo".into(), pubkey(42))
+                       .supported_quantity(Quantity::Bounded(ten))
+                       .supported_quantity(Quantity::one())
+                       .build()
+                       .unwrap();
+               let tlv_stream = offer.as_tlv_stream();
+               assert_eq!(offer.supported_quantity(), Quantity::one());
+               assert_eq!(tlv_stream.quantity_max, None);
+       }
+}
index 44967eb409be66ba8a3f11c900fc080bc148c27a..8a8d9924f9fb0ce33fcaca28602ff29a8598816d 100644 (file)
@@ -9,44 +9,52 @@
 
 //! Creating blinded routes and related utilities live here.
 
-use bitcoin::secp256k1::{self, PublicKey, Secp256k1, SecretKey};
+use bitcoin::hashes::{Hash, HashEngine};
+use bitcoin::hashes::sha256::Hash as Sha256;
+use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey};
 
-use crate::chain::keysinterface::KeysInterface;
+use crate::chain::keysinterface::{KeysInterface, Recipient};
+use super::packet::ControlTlvs;
 use super::utils;
 use crate::ln::msgs::DecodeError;
-use crate::util::chacha20poly1305rfc::ChaChaPolyWriteAdapter;
-use crate::util::ser::{Readable, VecWriter, Writeable, Writer};
+use crate::ln::onion_utils;
+use crate::util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter};
+use crate::util::ser::{FixedLengthReader, LengthReadableArgs, Readable, VecWriter, Writeable, Writer};
 
-use crate::io;
+use core::mem;
+use core::ops::Deref;
+use crate::io::{self, Cursor};
 use crate::prelude::*;
 
 /// Onion messages can be sent and received to blinded routes, which serve to hide the identity of
 /// the recipient.
+#[derive(Clone, Debug, PartialEq)]
 pub struct BlindedRoute {
        /// To send to a blinded route, the sender first finds a route to the unblinded
        /// `introduction_node_id`, which can unblind its [`encrypted_payload`] to find out the onion
        /// message's next hop and forward it along.
        ///
        /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(super) introduction_node_id: PublicKey,
+       pub(crate) introduction_node_id: PublicKey,
        /// Used by the introduction node to decrypt its [`encrypted_payload`] to forward the onion
        /// message.
        ///
        /// [`encrypted_payload`]: BlindedHop::encrypted_payload
-       pub(super) blinding_point: PublicKey,
+       pub(crate) blinding_point: PublicKey,
        /// The hops composing the blinded route.
-       pub(super) blinded_hops: Vec<BlindedHop>,
+       pub(crate) blinded_hops: Vec<BlindedHop>,
 }
 
 /// Used to construct the blinded hops portion of a blinded route. These hops cannot be identified
 /// by outside observers and thus can be used to hide the identity of the recipient.
+#[derive(Clone, Debug, PartialEq)]
 pub struct BlindedHop {
        /// The blinded node id of this hop in a blinded route.
-       pub(super) blinded_node_id: PublicKey,
+       pub(crate) blinded_node_id: PublicKey,
        /// The encrypted payload intended for this hop in a blinded route.
        // The node sending to this blinded route will later encode this payload into the onion packet for
        // this hop.
-       pub(super) encrypted_payload: Vec<u8>,
+       pub(crate) encrypted_payload: Vec<u8>,
 }
 
 impl BlindedRoute {
@@ -69,6 +77,41 @@ impl BlindedRoute {
                        blinded_hops: blinded_hops(secp_ctx, node_pks, &blinding_secret).map_err(|_| ())?,
                })
        }
+
+       // Advance the blinded route by one hop, so make the second hop into the new introduction node.
+       pub(super) fn advance_by_one<K: Deref, T: secp256k1::Signing + secp256k1::Verification>
+               (&mut self, keys_manager: &K, secp_ctx: &Secp256k1<T>) -> Result<(), ()>
+               where K::Target: KeysInterface
+       {
+               let control_tlvs_ss = keys_manager.ecdh(Recipient::Node, &self.blinding_point, None)?;
+               let rho = onion_utils::gen_rho_from_shared_secret(&control_tlvs_ss.secret_bytes());
+               let encrypted_control_tlvs = self.blinded_hops.remove(0).encrypted_payload;
+               let mut s = Cursor::new(&encrypted_control_tlvs);
+               let mut reader = FixedLengthReader::new(&mut s, encrypted_control_tlvs.len() as u64);
+               match ChaChaPolyReadAdapter::read(&mut reader, rho) {
+                       Ok(ChaChaPolyReadAdapter { readable: ControlTlvs::Forward(ForwardTlvs {
+                               mut next_node_id, next_blinding_override,
+                       })}) => {
+                               let mut new_blinding_point = match next_blinding_override {
+                                       Some(blinding_point) => blinding_point,
+                                       None => {
+                                               let blinding_factor = {
+                                                       let mut sha = Sha256::engine();
+                                                       sha.input(&self.blinding_point.serialize()[..]);
+                                                       sha.input(control_tlvs_ss.as_ref());
+                                                       Sha256::from_engine(sha).into_inner()
+                                               };
+                                               self.blinding_point.mul_tweak(secp_ctx, &Scalar::from_be_bytes(blinding_factor).unwrap())
+                                                       .map_err(|_| ())?
+                                       }
+                               };
+                               mem::swap(&mut self.blinding_point, &mut new_blinding_point);
+                               mem::swap(&mut self.introduction_node_id, &mut next_node_id);
+                               Ok(())
+                       },
+                       _ => Err(())
+               }
+       }
 }
 
 /// Construct blinded hops for the given `unblinded_path`.
index 5c623cf2f48a143a84a09c5eadc592d61ef21513..200570861c42bfa11566045eeafc31e988bc5ff7 100644 (file)
@@ -14,7 +14,7 @@ use crate::ln::features::InitFeatures;
 use crate::ln::msgs::{self, DecodeError, OnionMessageHandler};
 use super::{BlindedRoute, CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError};
 use crate::util::enforcing_trait_impls::EnforcingSigner;
-use crate::util::ser::{MaybeReadableArgs, Writeable, Writer};
+use crate::util::ser::{ Writeable, Writer};
 use crate::util::test_utils;
 
 use bitcoin::network::constants::Network;
@@ -25,7 +25,7 @@ use crate::sync::Arc;
 
 struct MessengerNode {
        keys_manager: Arc<test_utils::TestKeysInterface>,
-       messenger: OnionMessenger<EnforcingSigner, Arc<test_utils::TestKeysInterface>, Arc<test_utils::TestLogger>, Arc<TestCustomMessageHandler>>,
+       messenger: OnionMessenger<Arc<test_utils::TestKeysInterface>, Arc<test_utils::TestLogger>, Arc<TestCustomMessageHandler>>,
        logger: Arc<test_utils::TestLogger>,
 }
 
@@ -54,8 +54,12 @@ impl Writeable for TestCustomMessage {
        }
 }
 
-impl MaybeReadableArgs<u64> for TestCustomMessage {
-       fn read<R: io::Read>(buffer: &mut R, message_type: u64) -> Result<Option<Self>, DecodeError> where Self: Sized {
+struct TestCustomMessageHandler {}
+
+impl CustomOnionMessageHandler for TestCustomMessageHandler {
+       type CustomMessage = TestCustomMessage;
+       fn handle_custom_message(&self, _msg: Self::CustomMessage) {}
+       fn read_custom_message<R: io::Read>(&self, message_type: u64, buffer: &mut R) -> Result<Option<Self::CustomMessage>, DecodeError> where Self: Sized {
                if message_type == CUSTOM_MESSAGE_TYPE {
                        let mut buf = Vec::new();
                        buffer.read_to_end(&mut buf)?;
@@ -66,13 +70,6 @@ impl MaybeReadableArgs<u64> for TestCustomMessage {
        }
 }
 
-struct TestCustomMessageHandler {}
-
-impl CustomOnionMessageHandler for TestCustomMessageHandler {
-       type CustomMessage = TestCustomMessage;
-       fn handle_custom_message(&self, _msg: Self::CustomMessage) {}
-}
-
 fn create_nodes(num_messengers: u8) -> Vec<MessengerNode> {
        let mut nodes = Vec::new();
        for i in 0..num_messengers {
@@ -170,6 +167,26 @@ fn too_big_packet_error() {
        assert_eq!(err, SendError::TooBigPacket);
 }
 
+#[test]
+fn we_are_intro_node() {
+       // If we are sending straight to a blinded route and we are the introduction node, we need to
+       // advance the blinded route by 1 hop so the second hop is the new introduction node.
+       let mut nodes = create_nodes(3);
+       let test_msg = TestCustomMessage {};
+
+       let secp_ctx = Secp256k1::new();
+       let blinded_route = BlindedRoute::new(&[nodes[0].get_node_pk(), nodes[1].get_node_pk(), nodes[2].get_node_pk()], &*nodes[2].keys_manager, &secp_ctx).unwrap();
+
+       nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), OnionMessageContents::Custom(test_msg.clone()), None).unwrap();
+       pass_along_path(&nodes, None);
+
+       // Try with a two-hop blinded route where we are the introduction node.
+       let blinded_route = BlindedRoute::new(&[nodes[0].get_node_pk(), nodes[1].get_node_pk()], &*nodes[1].keys_manager, &secp_ctx).unwrap();
+       nodes[0].messenger.send_onion_message(&[], Destination::BlindedRoute(blinded_route), OnionMessageContents::Custom(test_msg), None).unwrap();
+       nodes.remove(2);
+       pass_along_path(&nodes, None);
+}
+
 #[test]
 fn invalid_blinded_route_error() {
        // Make sure we error as expected if a provided blinded route has 0 or 1 hops.
@@ -233,12 +250,6 @@ fn invalid_custom_message_type() {
                fn write<W: Writer>(&self, _w: &mut W) -> Result<(), io::Error> { unreachable!() }
        }
 
-       impl MaybeReadableArgs<u64> for InvalidCustomMessage {
-               fn read<R: io::Read>(_buffer: &mut R, _message_type: u64) -> Result<Option<Self>, DecodeError> where Self: Sized {
-                       unreachable!()
-               }
-       }
-
        let test_msg = OnionMessageContents::Custom(InvalidCustomMessage {});
        let err = nodes[0].messenger.send_onion_message(&[], Destination::Node(nodes[1].get_node_pk()), test_msg, None).unwrap_err();
        assert_eq!(err, SendError::InvalidMessage);
index ff20f19a7fc16a6bfea57e59f1b00eca27dc1d77..b884c2ffebd9f90abdbb6051e83e5c802aad4f8a 100644 (file)
@@ -15,7 +15,7 @@ use bitcoin::hashes::hmac::{Hmac, HmacEngine};
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::secp256k1::{self, PublicKey, Scalar, Secp256k1, SecretKey};
 
-use crate::chain::keysinterface::{InMemorySigner, KeysInterface, KeysManager, Recipient, Sign};
+use crate::chain::keysinterface::{KeysInterface, KeysManager, Recipient};
 use crate::ln::features::{InitFeatures, NodeFeatures};
 use crate::ln::msgs::{self, OnionMessageHandler};
 use crate::ln::onion_utils;
@@ -29,6 +29,7 @@ use crate::util::logger::Logger;
 use crate::util::ser::Writeable;
 
 use core::ops::Deref;
+use crate::io;
 use crate::sync::{Arc, Mutex};
 use crate::prelude::*;
 
@@ -47,7 +48,7 @@ use crate::prelude::*;
 /// # use lightning::ln::peer_handler::IgnoringMessageHandler;
 /// # use lightning::onion_message::{BlindedRoute, CustomOnionMessageContents, Destination, OnionMessageContents, OnionMessenger};
 /// # use lightning::util::logger::{Logger, Record};
-/// # use lightning::util::ser::{MaybeReadableArgs, Writeable, Writer};
+/// # use lightning::util::ser::{Writeable, Writer};
 /// # use lightning::io;
 /// # use std::sync::Arc;
 /// # struct FakeLogger {};
@@ -81,13 +82,6 @@ use crate::prelude::*;
 ///            your_custom_message_type
 ///    }
 /// }
-/// impl MaybeReadableArgs<u64> for YourCustomMessage {
-///    fn read<R: io::Read>(r: &mut R, message_type: u64) -> Result<Option<Self>, DecodeError> {
-///            # unreachable!()
-///            // Read your custom onion message of type `message_type` from `r`, or return `None`
-///            // if the message type is unknown
-///    }
-/// }
 /// // Send a custom onion message to a node id.
 /// let intermediate_hops = [hop_node_id1, hop_node_id2];
 /// let reply_path = None;
@@ -110,8 +104,8 @@ use crate::prelude::*;
 ///
 /// [offers]: <https://github.com/lightning/bolts/pull/798>
 /// [`OnionMessenger`]: crate::onion_message::OnionMessenger
-pub struct OnionMessenger<Signer: Sign, K: Deref, L: Deref, CMH: Deref>
-       where K::Target: KeysInterface<Signer = Signer>,
+pub struct OnionMessenger<K: Deref, L: Deref, CMH: Deref>
+       where K::Target: KeysInterface,
              L::Target: Logger,
              CMH:: Target: CustomOnionMessageHandler,
 {
@@ -160,6 +154,15 @@ pub enum SendError {
        InvalidMessage,
        /// Our next-hop peer's buffer was full or our total outbound buffer was full.
        BufferFull,
+       /// Failed to retrieve our node id from the provided [`KeysInterface`].
+       ///
+       /// [`KeysInterface`]: crate::chain::keysinterface::KeysInterface
+       GetNodeIdFailed,
+       /// We attempted to send to a blinded route where we are the introduction node, and failed to
+       /// advance the blinded route to make the second hop the new introduction node. Either
+       /// [`KeysInterface::ecdh`] failed, we failed to tweak the current blinding point to get the
+       /// new blinding point, or we were attempting to send to ourselves.
+       BlindedRouteAdvanceFailed,
 }
 
 /// Handler for custom onion messages. If you are using [`SimpleArcOnionMessenger`],
@@ -178,10 +181,13 @@ pub trait CustomOnionMessageHandler {
        type CustomMessage: CustomOnionMessageContents;
        /// Called with the custom message that was received.
        fn handle_custom_message(&self, msg: Self::CustomMessage);
+       /// Read a custom message of type `message_type` from `buffer`, returning `Ok(None)` if the
+       /// message type is unknown.
+       fn read_custom_message<R: io::Read>(&self, message_type: u64, buffer: &mut R) -> Result<Option<Self::CustomMessage>, msgs::DecodeError>;
 }
 
-impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessenger<Signer, K, L, CMH>
-       where K::Target: KeysInterface<Signer = Signer>,
+impl<K: Deref, L: Deref, CMH: Deref> OnionMessenger<K, L, CMH>
+       where K::Target: KeysInterface,
              L::Target: Logger,
              CMH::Target: CustomOnionMessageHandler,
 {
@@ -201,7 +207,7 @@ impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessenger<Signer, K, L,
 
        /// Send an onion message with contents `message` to `destination`, routing it through `intermediate_nodes`.
        /// See [`OnionMessenger`] for example usage.
-       pub fn send_onion_message<T: CustomOnionMessageContents>(&self, intermediate_nodes: &[PublicKey], destination: Destination, message: OnionMessageContents<T>, reply_path: Option<BlindedRoute>) -> Result<(), SendError> {
+       pub fn send_onion_message<T: CustomOnionMessageContents>(&self, intermediate_nodes: &[PublicKey], mut destination: Destination, message: OnionMessageContents<T>, reply_path: Option<BlindedRoute>) -> Result<(), SendError> {
                if let Destination::BlindedRoute(BlindedRoute { ref blinded_hops, .. }) = destination {
                        if blinded_hops.len() < 2 {
                                return Err(SendError::TooFewBlindedHops);
@@ -210,6 +216,19 @@ impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessenger<Signer, K, L,
                let OnionMessageContents::Custom(ref msg) = message;
                if msg.tlv_type() < 64 { return Err(SendError::InvalidMessage) }
 
+               // If we are sending straight to a blinded route and we are the introduction node, we need to
+               // advance the blinded route by 1 hop so the second hop is the new introduction node.
+               if intermediate_nodes.len() == 0 {
+                       if let Destination::BlindedRoute(ref mut blinded_route) = destination {
+                               let our_node_id = self.keys_manager.get_node_id(Recipient::Node)
+                                       .map_err(|()| SendError::GetNodeIdFailed)?;
+                               if blinded_route.introduction_node_id == our_node_id {
+                                       blinded_route.advance_by_one(&self.keys_manager, &self.secp_ctx)
+                                               .map_err(|()| SendError::BlindedRouteAdvanceFailed)?;
+                               }
+                       }
+               }
+
                let blinding_secret_bytes = self.keys_manager.get_secure_random_bytes();
                let blinding_secret = SecretKey::from_slice(&blinding_secret_bytes[..]).expect("RNG is busted");
                let (introduction_node_id, blinding_point) = if intermediate_nodes.len() != 0 {
@@ -276,10 +295,10 @@ fn outbound_buffer_full(peer_node_id: &PublicKey, buffer: &HashMap<PublicKey, Ve
        false
 }
 
-impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessageHandler for OnionMessenger<Signer, K, L, CMH>
-       where K::Target: KeysInterface<Signer = Signer>,
+impl<K: Deref, L: Deref, CMH: Deref> OnionMessageHandler for OnionMessenger<K, L, CMH>
+       where K::Target: KeysInterface,
              L::Target: Logger,
-             CMH::Target: CustomOnionMessageHandler,
+             CMH::Target: CustomOnionMessageHandler + Sized,
 {
        /// Handle an incoming onion message. Currently, if a message was destined for us we will log, but
        /// soon we'll delegate the onion message to a handler that can generate invoices or send
@@ -308,8 +327,8 @@ impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessageHandler for Onion
                                }
                        }
                };
-               match onion_utils::decode_next_hop(onion_decode_ss, &msg.onion_routing_packet.hop_data[..],
-                       msg.onion_routing_packet.hmac, control_tlvs_ss)
+               match onion_utils::decode_next_untagged_hop(onion_decode_ss, &msg.onion_routing_packet.hop_data[..],
+                       msg.onion_routing_packet.hmac, (control_tlvs_ss, &*self.custom_handler))
                {
                        Ok((Payload::Receive::<<<CMH as Deref>::Target as CustomOnionMessageHandler>::CustomMessage> {
                                message, control_tlvs: ReceiveControlTlvs::Unblinded(ReceiveTlvs { path_id }), reply_path,
@@ -420,8 +439,8 @@ impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessageHandler for Onion
        }
 }
 
-impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessageProvider for OnionMessenger<Signer, K, L, CMH>
-       where K::Target: KeysInterface<Signer = Signer>,
+impl<K: Deref, L: Deref, CMH: Deref> OnionMessageProvider for OnionMessenger<K, L, CMH>
+       where K::Target: KeysInterface,
              L::Target: Logger,
              CMH::Target: CustomOnionMessageHandler,
 {
@@ -443,7 +462,7 @@ impl<Signer: Sign, K: Deref, L: Deref, CMH: Deref> OnionMessageProvider for Onio
 ///
 /// [`SimpleArcChannelManager`]: crate::ln::channelmanager::SimpleArcChannelManager
 /// [`SimpleArcPeerManager`]: crate::ln::peer_handler::SimpleArcPeerManager
-pub type SimpleArcOnionMessenger<L> = OnionMessenger<InMemorySigner, Arc<KeysManager>, Arc<L>, IgnoringMessageHandler>;
+pub type SimpleArcOnionMessenger<L> = OnionMessenger<Arc<KeysManager>, Arc<L>, IgnoringMessageHandler>;
 /// Useful for simplifying the parameters of [`SimpleRefChannelManager`] and
 /// [`SimpleRefPeerManager`]. See their docs for more details.
 ///
@@ -451,7 +470,7 @@ pub type SimpleArcOnionMessenger<L> = OnionMessenger<InMemorySigner, Arc<KeysMan
 ///
 /// [`SimpleRefChannelManager`]: crate::ln::channelmanager::SimpleRefChannelManager
 /// [`SimpleRefPeerManager`]: crate::ln::peer_handler::SimpleRefPeerManager
-pub type SimpleRefOnionMessenger<'a, 'b, L> = OnionMessenger<InMemorySigner, &'a KeysManager, &'b L, IgnoringMessageHandler>;
+pub type SimpleRefOnionMessenger<'a, 'b, L> = OnionMessenger<&'a KeysManager, &'b L, IgnoringMessageHandler>;
 
 /// Construct onion packet payloads and keys for sending an onion message along the given
 /// `unblinded_path` to the given `destination`.
@@ -491,12 +510,8 @@ fn packet_payloads_and_keys<T: CustomOnionMessageContents, S: secp256k1::Signing
                                        next_blinding_override: Some(blinding_pt),
                                })), control_tlvs_ss));
                        }
-                       if let Some(encrypted_payload) = enc_payload_opt {
-                               payloads.push((Payload::Forward(ForwardControlTlvs::Blinded(encrypted_payload)),
-                                       control_tlvs_ss));
-                       } else { debug_assert!(false); }
-                       blinded_path_idx += 1;
-               } else if blinded_path_idx < num_blinded_hops - 1 && enc_payload_opt.is_some() {
+               }
+               if blinded_path_idx < num_blinded_hops.saturating_sub(1) && enc_payload_opt.is_some() {
                        payloads.push((Payload::Forward(ForwardControlTlvs::Blinded(enc_payload_opt.unwrap())),
                                control_tlvs_ss));
                        blinded_path_idx += 1;
index 3c522e30c51bf367c25311b6987418f3fe802931..89c1545a7df29c59adfe0956bc9faf1f769e9b80 100644 (file)
@@ -28,6 +28,6 @@ mod utils;
 mod functional_tests;
 
 // Re-export structs so they can be imported with just the `onion_message::` module prefix.
-pub use self::blinded_route::{BlindedRoute, BlindedHop};
+pub use self::blinded_route::{BlindedRoute, BlindedRoute as BlindedPath, BlindedHop};
 pub use self::messenger::{CustomOnionMessageContents, CustomOnionMessageHandler, Destination, OnionMessageContents, OnionMessenger, SendError, SimpleArcOnionMessenger, SimpleRefOnionMessenger};
 pub(crate) use self::packet::Packet;
index da8dd561072edd516182b377229788518d431e77..8fff53642e128682638ec187eab0c8a31afca260 100644 (file)
@@ -15,8 +15,9 @@ use bitcoin::secp256k1::ecdh::SharedSecret;
 use crate::ln::msgs::DecodeError;
 use crate::ln::onion_utils;
 use super::blinded_route::{BlindedRoute, ForwardTlvs, ReceiveTlvs};
+use super::messenger::CustomOnionMessageHandler;
 use crate::util::chacha20poly1305rfc::{ChaChaPolyReadAdapter, ChaChaPolyWriteAdapter};
-use crate::util::ser::{BigSize, FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, MaybeReadableArgs, Readable, ReadableArgs, Writeable, Writer};
+use crate::util::ser::{BigSize, FixedLengthReader, LengthRead, LengthReadable, LengthReadableArgs, Readable, ReadableArgs, Writeable, Writer};
 
 use core::cmp;
 use crate::io::{self, Read};
@@ -106,7 +107,7 @@ pub(super) enum Payload<T: CustomOnionMessageContents> {
 #[derive(Debug)]
 /// The contents of an onion message. In the context of offers, this would be the invoice, invoice
 /// request, or invoice error.
-pub enum OnionMessageContents<T> where T: CustomOnionMessageContents {
+pub enum OnionMessageContents<T: CustomOnionMessageContents> {
        // Coming soon:
        // Invoice,
        // InvoiceRequest,
@@ -115,7 +116,7 @@ pub enum OnionMessageContents<T> where T: CustomOnionMessageContents {
        Custom(T),
 }
 
-impl<T> OnionMessageContents<T> where T: CustomOnionMessageContents {
+impl<T: CustomOnionMessageContents> OnionMessageContents<T> {
        /// Returns the type that was used to decode the message payload.
        pub fn tlv_type(&self) -> u64 {
                match self {
@@ -132,9 +133,8 @@ impl<T: CustomOnionMessageContents> Writeable for OnionMessageContents<T> {
        }
 }
 
-/// The contents of a custom onion message. Must implement `MaybeReadableArgs<u64>` where the `u64`
-/// is the custom TLV type attempting to be read, and return `Ok(None)` if the TLV type is unknown.
-pub trait CustomOnionMessageContents: Writeable + MaybeReadableArgs<u64> {
+/// The contents of a custom onion message.
+pub trait CustomOnionMessageContents: Writeable {
        /// Returns the TLV type identifying the message contents. MUST be >= 64.
        fn tlv_type(&self) -> u64;
 }
@@ -164,7 +164,7 @@ impl<T: CustomOnionMessageContents> Writeable for (Payload<T>, [u8; 32]) {
                match &self.0 {
                        Payload::Forward(ForwardControlTlvs::Blinded(encrypted_bytes)) => {
                                encode_varint_length_prefixed_tlv!(w, {
-                                       (4, encrypted_bytes, vec_type)
+                                       (4, *encrypted_bytes, vec_type)
                                })
                        },
                        Payload::Receive {
@@ -172,7 +172,7 @@ impl<T: CustomOnionMessageContents> Writeable for (Payload<T>, [u8; 32]) {
                        } => {
                                encode_varint_length_prefixed_tlv!(w, {
                                        (2, reply_path, option),
-                                       (4, encrypted_bytes, vec_type),
+                                       (4, *encrypted_bytes, vec_type),
                                        (message.tlv_type(), message, required)
                                })
                        },
@@ -198,8 +198,10 @@ impl<T: CustomOnionMessageContents> Writeable for (Payload<T>, [u8; 32]) {
 }
 
 // Uses the provided secret to simultaneously decode and decrypt the control TLVs and data TLV.
-impl<T: CustomOnionMessageContents> ReadableArgs<SharedSecret> for Payload<T> {
-       fn read<R: Read>(r: &mut R, encrypted_tlvs_ss: SharedSecret) -> Result<Self, DecodeError> {
+impl<H: CustomOnionMessageHandler> ReadableArgs<(SharedSecret, &H)> for Payload<<H as CustomOnionMessageHandler>::CustomMessage> {
+       fn read<R: Read>(r: &mut R, args: (SharedSecret, &H)) -> Result<Self, DecodeError> {
+               let (encrypted_tlvs_ss, handler) = args;
+
                let v: BigSize = Readable::read(r)?;
                let mut rd = FixedLengthReader::new(r, v.0);
                let mut reply_path: Option<BlindedRoute> = None;
@@ -216,7 +218,7 @@ impl<T: CustomOnionMessageContents> ReadableArgs<SharedSecret> for Payload<T> {
                        if message_type.is_some() { return Err(DecodeError::InvalidValue) }
 
                        message_type = Some(msg_type);
-                       match T::read(msg_reader, msg_type) {
+                       match handler.read_custom_message(msg_type, msg_reader) {
                                Ok(Some(msg)) => {
                                        message = Some(msg);
                                        Ok(true)
index 1706e0692ef90ea1fdfbe26e7387239cfd188d3f..947df9edae0f2b3a166e96ff704c9c5775e7da8e 100644 (file)
@@ -29,8 +29,9 @@ use crate::ln::msgs::{QueryChannelRange, ReplyChannelRange, QueryShortChannelIds
 use crate::ln::msgs;
 use crate::util::ser::{Readable, ReadableArgs, Writeable, Writer, MaybeReadable};
 use crate::util::logger::{Logger, Level};
-use crate::util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider};
+use crate::util::events::{MessageSendEvent, MessageSendEventsProvider};
 use crate::util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK};
+use crate::util::string::PrintableString;
 
 use crate::io;
 use crate::io_extras::{copy, sink};
@@ -212,9 +213,6 @@ impl_writeable_tlv_based_enum_upgradable!(NetworkUpdate,
 /// This network graph is then used for routing payments.
 /// Provides interface to help with initial routing sync by
 /// serving historical announcements.
-///
-/// Serves as an [`EventHandler`] for applying updates from [`Event::PaymentPathFailed`] to the
-/// [`NetworkGraph`].
 pub struct P2PGossipSync<G: Deref<Target=NetworkGraph<L>>, C: Deref, L: Deref>
 where C::Target: chain::Access, L::Target: Logger
 {
@@ -274,32 +272,31 @@ where C::Target: chain::Access, L::Target: Logger
        }
 }
 
-impl<L: Deref> EventHandler for NetworkGraph<L> where L::Target: Logger {
-       fn handle_event(&self, event: &Event) {
-               if let Event::PaymentPathFailed { network_update, .. } = event {
-                       if let Some(network_update) = network_update {
-                               match *network_update {
-                                       NetworkUpdate::ChannelUpdateMessage { ref msg } => {
-                                               let short_channel_id = msg.contents.short_channel_id;
-                                               let is_enabled = msg.contents.flags & (1 << 1) != (1 << 1);
-                                               let status = if is_enabled { "enabled" } else { "disabled" };
-                                               log_debug!(self.logger, "Updating channel with channel_update from a payment failure. Channel {} is {}.", short_channel_id, status);
-                                               let _ = self.update_channel(msg);
-                                       },
-                                       NetworkUpdate::ChannelFailure { short_channel_id, is_permanent } => {
-                                               let action = if is_permanent { "Removing" } else { "Disabling" };
-                                               log_debug!(self.logger, "{} channel graph entry for {} due to a payment failure.", action, short_channel_id);
-                                               self.channel_failed(short_channel_id, is_permanent);
-                                       },
-                                       NetworkUpdate::NodeFailure { ref node_id, is_permanent } => {
-                                               if is_permanent {
-                                                       log_debug!(self.logger,
-                                                               "Removed node graph entry for {} due to a payment failure.", log_pubkey!(node_id));
-                                                       self.node_failed_permanent(node_id);
-                                               };
-                                       },
-                               }
-                       }
+impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
+       /// Handles any network updates originating from [`Event`]s.
+       ///
+       /// [`Event`]: crate::util::events::Event
+       pub fn handle_network_update(&self, network_update: &NetworkUpdate) {
+               match *network_update {
+                       NetworkUpdate::ChannelUpdateMessage { ref msg } => {
+                               let short_channel_id = msg.contents.short_channel_id;
+                               let is_enabled = msg.contents.flags & (1 << 1) != (1 << 1);
+                               let status = if is_enabled { "enabled" } else { "disabled" };
+                               log_debug!(self.logger, "Updating channel with channel_update from a payment failure. Channel {} is {}.", short_channel_id, status);
+                               let _ = self.update_channel(msg);
+                       },
+                       NetworkUpdate::ChannelFailure { short_channel_id, is_permanent } => {
+                               let action = if is_permanent { "Removing" } else { "Disabling" };
+                               log_debug!(self.logger, "{} channel graph entry for {} due to a payment failure.", action, short_channel_id);
+                               self.channel_failed(short_channel_id, is_permanent);
+                       },
+                       NetworkUpdate::NodeFailure { ref node_id, is_permanent } => {
+                               if is_permanent {
+                                       log_debug!(self.logger,
+                                               "Removed node graph entry for {} due to a payment failure.", log_pubkey!(node_id));
+                                       self.node_failed_permanent(node_id);
+                               };
+                       },
                }
        }
 }
@@ -750,7 +747,7 @@ impl ChannelInfo {
                                return None;
                        }
                };
-               Some((DirectedChannelInfo::new(self, direction), source))
+               direction.map(|dir| (DirectedChannelInfo::new(self, dir), source))
        }
 
        /// Returns a [`DirectedChannelInfo`] for the channel directed from the given `source` to a
@@ -765,7 +762,7 @@ impl ChannelInfo {
                                return None;
                        }
                };
-               Some((DirectedChannelInfo::new(self, direction), target))
+               direction.map(|dir| (DirectedChannelInfo::new(self, dir), target))
        }
 
        /// Returns a [`ChannelUpdateInfo`] based on the direction implied by the channel_flag.
@@ -860,29 +857,23 @@ impl Readable for ChannelInfo {
 #[derive(Clone)]
 pub struct DirectedChannelInfo<'a> {
        channel: &'a ChannelInfo,
-       direction: Option<&'a ChannelUpdateInfo>,
+       direction: &'a ChannelUpdateInfo,
        htlc_maximum_msat: u64,
        effective_capacity: EffectiveCapacity,
 }
 
 impl<'a> DirectedChannelInfo<'a> {
        #[inline]
-       fn new(channel: &'a ChannelInfo, direction: Option<&'a ChannelUpdateInfo>) -> Self {
-               let htlc_maximum_msat = direction.map(|direction| direction.htlc_maximum_msat);
+       fn new(channel: &'a ChannelInfo, direction: &'a ChannelUpdateInfo) -> Self {
+               let mut htlc_maximum_msat = direction.htlc_maximum_msat;
                let capacity_msat = channel.capacity_sats.map(|capacity_sats| capacity_sats * 1000);
 
-               let (htlc_maximum_msat, effective_capacity) = match (htlc_maximum_msat, capacity_msat) {
-                       (Some(amount_msat), Some(capacity_msat)) => {
-                               let htlc_maximum_msat = cmp::min(amount_msat, capacity_msat);
-                               (htlc_maximum_msat, EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: Some(htlc_maximum_msat) })
-                       },
-                       (Some(amount_msat), None) => {
-                               (amount_msat, EffectiveCapacity::MaximumHTLC { amount_msat })
-                       },
-                       (None, Some(capacity_msat)) => {
-                               (capacity_msat, EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: None })
+               let effective_capacity = match capacity_msat {
+                       Some(capacity_msat) => {
+                               htlc_maximum_msat = cmp::min(htlc_maximum_msat, capacity_msat);
+                               EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: htlc_maximum_msat }
                        },
-                       (None, None) => (EffectiveCapacity::Unknown.as_msat(), EffectiveCapacity::Unknown),
+                       None => EffectiveCapacity::MaximumHTLC { amount_msat: htlc_maximum_msat },
                };
 
                Self {
@@ -891,12 +882,11 @@ impl<'a> DirectedChannelInfo<'a> {
        }
 
        /// Returns information for the channel.
+       #[inline]
        pub fn channel(&self) -> &'a ChannelInfo { self.channel }
 
-       /// Returns information for the direction.
-       pub fn direction(&self) -> Option<&'a ChannelUpdateInfo> { self.direction }
-
        /// Returns the maximum HTLC amount allowed over the channel in the direction.
+       #[inline]
        pub fn htlc_maximum_msat(&self) -> u64 {
                self.htlc_maximum_msat
        }
@@ -910,13 +900,9 @@ impl<'a> DirectedChannelInfo<'a> {
                self.effective_capacity
        }
 
-       /// Returns `Some` if [`ChannelUpdateInfo`] is available in the direction.
-       pub(super) fn with_update(self) -> Option<DirectedChannelInfoWithUpdate<'a>> {
-               match self.direction {
-                       Some(_) => Some(DirectedChannelInfoWithUpdate { inner: self }),
-                       None => None,
-               }
-       }
+       /// Returns information for the direction.
+       #[inline]
+       pub(super) fn direction(&self) -> &'a ChannelUpdateInfo { self.direction }
 }
 
 impl<'a> fmt::Debug for DirectedChannelInfo<'a> {
@@ -927,32 +913,6 @@ impl<'a> fmt::Debug for DirectedChannelInfo<'a> {
        }
 }
 
-/// A [`DirectedChannelInfo`] with [`ChannelUpdateInfo`] available in its direction.
-#[derive(Clone)]
-pub(super) struct DirectedChannelInfoWithUpdate<'a> {
-       inner: DirectedChannelInfo<'a>,
-}
-
-impl<'a> DirectedChannelInfoWithUpdate<'a> {
-       /// Returns information for the channel.
-       #[inline]
-       pub(super) fn channel(&self) -> &'a ChannelInfo { &self.inner.channel }
-
-       /// Returns information for the direction.
-       #[inline]
-       pub(super) fn direction(&self) -> &'a ChannelUpdateInfo { self.inner.direction.unwrap() }
-
-       /// Returns the [`EffectiveCapacity`] of the channel in the direction.
-       #[inline]
-       pub(super) fn effective_capacity(&self) -> EffectiveCapacity { self.inner.effective_capacity() }
-}
-
-impl<'a> fmt::Debug for DirectedChannelInfoWithUpdate<'a> {
-       fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-               self.inner.fmt(f)
-       }
-}
-
 /// The effective capacity of a channel for routing purposes.
 ///
 /// While this may be smaller than the actual channel capacity, amounts greater than
@@ -976,7 +936,7 @@ pub enum EffectiveCapacity {
                /// The funding amount denominated in millisatoshi.
                capacity_msat: u64,
                /// The maximum HTLC amount denominated in millisatoshi.
-               htlc_maximum_msat: Option<u64>
+               htlc_maximum_msat: u64
        },
        /// A capacity sufficient to route any payment, typically used for private channels provided by
        /// an invoice.
@@ -1059,23 +1019,17 @@ pub struct NodeAlias(pub [u8; 32]);
 
 impl fmt::Display for NodeAlias {
        fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
-               let control_symbol = core::char::REPLACEMENT_CHARACTER;
                let first_null = self.0.iter().position(|b| *b == 0).unwrap_or(self.0.len());
                let bytes = self.0.split_at(first_null).0;
                match core::str::from_utf8(bytes) {
-                       Ok(alias) => {
-                               for c in alias.chars() {
-                                       let mut bytes = [0u8; 4];
-                                       let c = if !c.is_control() { c } else { control_symbol };
-                                       f.write_str(c.encode_utf8(&mut bytes))?;
-                               }
-                       },
+                       Ok(alias) => PrintableString(alias).fmt(f)?,
                        Err(_) => {
+                               use core::fmt::Write;
                                for c in bytes.iter().map(|b| *b as char) {
                                        // Display printable ASCII characters
-                                       let mut bytes = [0u8; 4];
+                                       let control_symbol = core::char::REPLACEMENT_CHARACTER;
                                        let c = if c >= '\x20' && c <= '\x7e' { c } else { control_symbol };
-                                       f.write_str(c.encode_utf8(&mut bytes))?;
+                                       f.write_char(c)?;
                                }
                        },
                };
@@ -1690,7 +1644,7 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                        if info.two_to_one.is_some() && info.two_to_one.as_ref().unwrap().last_update < min_time_unix {
                                info.two_to_one = None;
                        }
-                       if info.one_to_two.is_none() && info.two_to_one.is_none() {
+                       if info.one_to_two.is_none() || info.two_to_one.is_none() {
                                // We check the announcement_received_time here to ensure we don't drop
                                // announcements that we just received and are just waiting for our peer to send a
                                // channel_update for.
@@ -1704,6 +1658,7 @@ impl<L: Deref> NetworkGraph<L> where L::Target: Logger {
                        for scid in scids_to_remove {
                                let info = channels.remove(&scid).expect("We just accessed this scid, it should be present");
                                Self::remove_channel_in_nodes(&mut nodes, &info, scid);
+                               self.removed_channels.lock().unwrap().insert(scid, Some(current_time_unix));
                        }
                }
 
@@ -1972,7 +1927,6 @@ mod tests {
        use crate::chain;
        use crate::ln::channelmanager;
        use crate::ln::chan_utils::make_funding_redeemscript;
-       use crate::ln::PaymentHash;
        use crate::ln::features::InitFeatures;
        use crate::routing::gossip::{P2PGossipSync, NetworkGraph, NetworkUpdate, NodeAlias, MAX_EXCESS_BYTES_FOR_RELAY, NodeId, RoutingFees, ChannelUpdateInfo, ChannelInfo, NodeAnnouncementInfo, NodeInfo};
        use crate::ln::msgs::{RoutingMessageHandler, UnsignedNodeAnnouncement, NodeAnnouncement,
@@ -1980,7 +1934,7 @@ mod tests {
                ReplyChannelRange, QueryChannelRange, QueryShortChannelIds, MAX_VALUE_MSAT};
        use crate::util::test_utils;
        use crate::util::ser::{ReadableArgs, Writeable};
-       use crate::util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider};
+       use crate::util::events::{MessageSendEvent, MessageSendEventsProvider};
        use crate::util::scid_utils::scid_from_parts;
 
        use crate::routing::gossip::REMOVED_ENTRIES_TRACKING_AGE_LIMIT_SECS;
@@ -2424,19 +2378,8 @@ mod tests {
                        let valid_channel_update = get_signed_channel_update(|_| {}, node_1_privkey, &secp_ctx);
                        assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_none());
 
-                       network_graph.handle_event(&Event::PaymentPathFailed {
-                               payment_id: None,
-                               payment_hash: PaymentHash([0; 32]),
-                               payment_failed_permanently: false,
-                               all_paths_failed: true,
-                               path: vec![],
-                               network_update: Some(NetworkUpdate::ChannelUpdateMessage {
-                                       msg: valid_channel_update,
-                               }),
-                               short_channel_id: None,
-                               retry: None,
-                               error_code: None,
-                               error_data: None,
+                       network_graph.handle_network_update(&NetworkUpdate::ChannelUpdateMessage {
+                               msg: valid_channel_update,
                        });
 
                        assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_some());
@@ -2451,20 +2394,9 @@ mod tests {
                                }
                        };
 
-                       network_graph.handle_event(&Event::PaymentPathFailed {
-                               payment_id: None,
-                               payment_hash: PaymentHash([0; 32]),
-                               payment_failed_permanently: false,
-                               all_paths_failed: true,
-                               path: vec![],
-                               network_update: Some(NetworkUpdate::ChannelFailure {
-                                       short_channel_id,
-                                       is_permanent: false,
-                               }),
-                               short_channel_id: None,
-                               retry: None,
-                               error_code: None,
-                               error_data: None,
+                       network_graph.handle_network_update(&NetworkUpdate::ChannelFailure {
+                               short_channel_id,
+                               is_permanent: false,
                        });
 
                        match network_graph.read_only().channels().get(&short_channel_id) {
@@ -2476,20 +2408,9 @@ mod tests {
                }
 
                // Permanent closing deletes a channel
-               network_graph.handle_event(&Event::PaymentPathFailed {
-                       payment_id: None,
-                       payment_hash: PaymentHash([0; 32]),
-                       payment_failed_permanently: false,
-                       all_paths_failed: true,
-                       path: vec![],
-                       network_update: Some(NetworkUpdate::ChannelFailure {
-                               short_channel_id,
-                               is_permanent: true,
-                       }),
-                       short_channel_id: None,
-                       retry: None,
-                       error_code: None,
-                       error_data: None,
+               network_graph.handle_network_update(&NetworkUpdate::ChannelFailure {
+                       short_channel_id,
+                       is_permanent: true,
                });
 
                assert_eq!(network_graph.read_only().channels().len(), 0);
@@ -2508,40 +2429,18 @@ mod tests {
                        assert!(network_graph.read_only().channels().get(&short_channel_id).is_some());
 
                        // Non-permanent node failure does not delete any nodes or channels
-                       network_graph.handle_event(&Event::PaymentPathFailed {
-                               payment_id: None,
-                               payment_hash: PaymentHash([0; 32]),
-                               payment_failed_permanently: false,
-                               all_paths_failed: true,
-                               path: vec![],
-                               network_update: Some(NetworkUpdate::NodeFailure {
-                                       node_id: node_2_id,
-                                       is_permanent: false,
-                               }),
-                               short_channel_id: None,
-                               retry: None,
-                               error_code: None,
-                               error_data: None,
+                       network_graph.handle_network_update(&NetworkUpdate::NodeFailure {
+                               node_id: node_2_id,
+                               is_permanent: false,
                        });
 
                        assert!(network_graph.read_only().channels().get(&short_channel_id).is_some());
                        assert!(network_graph.read_only().nodes().get(&NodeId::from_pubkey(&node_2_id)).is_some());
 
                        // Permanent node failure deletes node and its channels
-                       network_graph.handle_event(&Event::PaymentPathFailed {
-                               payment_id: None,
-                               payment_hash: PaymentHash([0; 32]),
-                               payment_failed_permanently: false,
-                               all_paths_failed: true,
-                               path: vec![],
-                               network_update: Some(NetworkUpdate::NodeFailure {
-                                       node_id: node_2_id,
-                                       is_permanent: true,
-                               }),
-                               short_channel_id: None,
-                               retry: None,
-                               error_code: None,
-                               error_data: None,
+                       network_graph.handle_network_update(&NetworkUpdate::NodeFailure {
+                               node_id: node_2_id,
+                               is_permanent: true,
                        });
 
                        assert_eq!(network_graph.read_only().nodes().len(), 0);
@@ -2569,32 +2468,57 @@ mod tests {
                assert!(network_graph.update_channel_from_announcement(&valid_channel_announcement, &chain_source).is_ok());
                assert!(network_graph.read_only().channels().get(&short_channel_id).is_some());
 
+               // Submit two channel updates for each channel direction (update.flags bit).
                let valid_channel_update = get_signed_channel_update(|_| {}, node_1_privkey, &secp_ctx);
                assert!(gossip_sync.handle_channel_update(&valid_channel_update).is_ok());
                assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_some());
 
+               let valid_channel_update_2 = get_signed_channel_update(|update| {update.flags |=1;}, node_2_privkey, &secp_ctx);
+               gossip_sync.handle_channel_update(&valid_channel_update_2).unwrap();
+               assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().two_to_one.is_some());
+
                network_graph.remove_stale_channels_and_tracking_with_time(100 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS);
                assert_eq!(network_graph.read_only().channels().len(), 1);
                assert_eq!(network_graph.read_only().nodes().len(), 2);
 
                network_graph.remove_stale_channels_and_tracking_with_time(101 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS);
+               #[cfg(not(feature = "std"))] {
+                       // Make sure removed channels are tracked.
+                       assert_eq!(network_graph.removed_channels.lock().unwrap().len(), 1);
+               }
+               network_graph.remove_stale_channels_and_tracking_with_time(101 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS +
+                       REMOVED_ENTRIES_TRACKING_AGE_LIMIT_SECS);
+
                #[cfg(feature = "std")]
                {
                        // In std mode, a further check is performed before fully removing the channel -
                        // the channel_announcement must have been received at least two weeks ago. We
-                       // fudge that here by indicating the time has jumped two weeks. Note that the
-                       // directional channel information will have been removed already..
+                       // fudge that here by indicating the time has jumped two weeks.
                        assert_eq!(network_graph.read_only().channels().len(), 1);
                        assert_eq!(network_graph.read_only().nodes().len(), 2);
-                       assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_none());
 
+                       // Note that the directional channel information will have been removed already..
+                       // We want to check that this will work even if *one* of the channel updates is recent,
+                       // so we should add it with a recent timestamp.
+                       assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_none());
                        use std::time::{SystemTime, UNIX_EPOCH};
                        let announcement_time = SystemTime::now().duration_since(UNIX_EPOCH).expect("Time must be > 1970").as_secs();
+                       let valid_channel_update = get_signed_channel_update(|unsigned_channel_update| {
+                               unsigned_channel_update.timestamp = (announcement_time + 1 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS) as u32;
+                       }, node_1_privkey, &secp_ctx);
+                       assert!(gossip_sync.handle_channel_update(&valid_channel_update).is_ok());
+                       assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_some());
                        network_graph.remove_stale_channels_and_tracking_with_time(announcement_time + 1 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS);
+                       // Make sure removed channels are tracked.
+                       assert_eq!(network_graph.removed_channels.lock().unwrap().len(), 1);
+                       // Provide a later time so that sufficient time has passed
+                       network_graph.remove_stale_channels_and_tracking_with_time(announcement_time + 1 + STALE_CHANNEL_UPDATE_AGE_LIMIT_SECS +
+                               REMOVED_ENTRIES_TRACKING_AGE_LIMIT_SECS);
                }
 
                assert_eq!(network_graph.read_only().channels().len(), 0);
                assert_eq!(network_graph.read_only().nodes().len(), 0);
+               assert!(network_graph.removed_channels.lock().unwrap().is_empty());
 
                #[cfg(feature = "std")]
                {
index 0d37744db83aabbd93ef4519b2f1ceef89ca84f8..cffd9ddb0a581ea2e26a53ee3f305a8c7f81bbf7 100644 (file)
@@ -17,7 +17,7 @@ use bitcoin::secp256k1::PublicKey;
 use crate::ln::channelmanager::ChannelDetails;
 use crate::ln::features::{ChannelFeatures, InvoiceFeatures, NodeFeatures};
 use crate::ln::msgs::{DecodeError, ErrorAction, LightningError, MAX_VALUE_MSAT};
-use crate::routing::gossip::{DirectedChannelInfoWithUpdate, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
+use crate::routing::gossip::{DirectedChannelInfo, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
 use crate::routing::scoring::{ChannelUsage, Score};
 use crate::util::ser::{Writeable, Readable, Writer};
 use crate::util::logger::{Level, Logger};
@@ -29,6 +29,49 @@ use alloc::collections::BinaryHeap;
 use core::cmp;
 use core::ops::Deref;
 
+/// A trait defining behavior for routing a payment.
+pub trait Router {
+       /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values.
+       fn find_route(
+               &self, payer: &PublicKey, route_params: &RouteParameters,
+               first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
+       ) -> Result<Route, LightningError>;
+}
+
+/// A map with liquidity value (in msat) keyed by a short channel id and the direction the HTLC
+/// is traveling in. The direction boolean is determined by checking if the HTLC source's public
+/// key is less than its destination. See [`InFlightHtlcs::used_liquidity_msat`] for more
+/// details.
+#[cfg(not(any(test, feature = "_test_utils")))]
+pub struct InFlightHtlcs(HashMap<(u64, bool), u64>);
+#[cfg(any(test, feature = "_test_utils"))]
+pub struct InFlightHtlcs(pub HashMap<(u64, bool), u64>);
+
+impl InFlightHtlcs {
+       /// Create a new `InFlightHtlcs` via a mapping from:
+       /// (short_channel_id, source_pubkey < target_pubkey) -> used_liquidity_msat
+       pub fn new(inflight_map: HashMap<(u64, bool), u64>) -> Self {
+               InFlightHtlcs(inflight_map)
+       }
+
+       /// Returns liquidity in msat given the public key of the HTLC source, target, and short channel
+       /// id.
+       pub fn used_liquidity_msat(&self, source: &NodeId, target: &NodeId, channel_scid: u64) -> Option<u64> {
+               self.0.get(&(channel_scid, source < target)).map(|v| *v)
+       }
+}
+
+impl Writeable for InFlightHtlcs {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> { self.0.write(writer) }
+}
+
+impl Readable for InFlightHtlcs {
+       fn read<R: io::Read>(reader: &mut R) -> Result<Self, DecodeError> {
+               let infight_map: HashMap<(u64, bool), u64> = Readable::read(reader)?;
+               Ok(Self(infight_map))
+       }
+}
+
 /// A hop in a route
 #[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct RouteHop {
@@ -185,8 +228,7 @@ pub const DEFAULT_MAX_PATH_COUNT: u8 = 10;
 const MEDIAN_HOP_CLTV_EXPIRY_DELTA: u32 = 40;
 
 // During routing, we only consider paths shorter than our maximum length estimate.
-// In the legacy onion format, the maximum number of hops used to be a fixed value of 20.
-// However, in the TLV onion format, there is no fixed maximum length, but the `hop_payloads`
+// In the TLV onion format, there is no fixed maximum length, but the `hop_payloads`
 // field is always 1300 bytes. As the `tlv_payload` for each hop may vary in length, we have to
 // estimate how many hops the route may have so that it actually fits the `hop_payloads` field.
 //
@@ -421,7 +463,7 @@ enum CandidateRouteHop<'a> {
        },
        /// A hop found in the [`ReadOnlyNetworkGraph`], where the channel capacity may be unknown.
        PublicHop {
-               info: DirectedChannelInfoWithUpdate<'a>,
+               info: DirectedChannelInfo<'a>,
                short_channel_id: u64,
        },
        /// A hop to the payee found in the payment invoice, though not necessarily a direct channel.
@@ -494,10 +536,8 @@ fn max_htlc_from_capacity(capacity: EffectiveCapacity, max_channel_saturation_po
                EffectiveCapacity::Unknown => EffectiveCapacity::Unknown.as_msat(),
                EffectiveCapacity::MaximumHTLC { amount_msat } =>
                        amount_msat.checked_shr(saturation_shift).unwrap_or(0),
-               EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: None } =>
-                       capacity_msat.checked_shr(saturation_shift).unwrap_or(0),
-               EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: Some(htlc_max) } =>
-                       cmp::min(capacity_msat.checked_shr(saturation_shift).unwrap_or(0), htlc_max),
+               EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat } =>
+                       cmp::min(capacity_msat.checked_shr(saturation_shift).unwrap_or(0), htlc_maximum_msat),
        }
 }
 
@@ -1276,13 +1316,11 @@ where L::Target: Logger {
                                        for chan_id in $node.channels.iter() {
                                                let chan = network_channels.get(chan_id).unwrap();
                                                if !chan.features.requires_unknown_bits() {
-                                                       let (directed_channel, source) =
-                                                               chan.as_directed_to(&$node_id).expect("inconsistent NetworkGraph");
-                                                       if first_hops.is_none() || *source != our_node_id {
-                                                               if let Some(direction) = directed_channel.direction() {
-                                                                       if direction.enabled {
+                                                       if let Some((directed_channel, source)) = chan.as_directed_to(&$node_id) {
+                                                               if first_hops.is_none() || *source != our_node_id {
+                                                                       if directed_channel.direction().enabled {
                                                                                let candidate = CandidateRouteHop::PublicHop {
-                                                                                       info: directed_channel.with_update().unwrap(),
+                                                                                       info: directed_channel,
                                                                                        short_channel_id: *chan_id,
                                                                                };
                                                                                add_entry!(candidate, *source, $node_id,
@@ -1367,8 +1405,7 @@ where L::Target: Logger {
                                        let candidate = network_channels
                                                .get(&hop.short_channel_id)
                                                .and_then(|channel| channel.as_directed_to(&target))
-                                               .and_then(|(channel, _)| channel.with_update())
-                                               .map(|info| CandidateRouteHop::PublicHop {
+                                               .map(|(info, _)| CandidateRouteHop::PublicHop {
                                                        info,
                                                        short_channel_id: hop.short_channel_id,
                                                })
@@ -1816,10 +1853,8 @@ fn add_random_cltv_offset(route: &mut Route, payment_params: &PaymentParameters,
                                                        random_channel.as_directed_from(&cur_node_id).map(|(dir_info, next_id)| {
                                                                if !nodes_to_avoid.iter().any(|x| x == next_id) {
                                                                        nodes_to_avoid[random_hop] = *next_id;
-                                                                       dir_info.direction().map(|channel_update_info| {
-                                                                               random_hop_offset = channel_update_info.cltv_expiry_delta.into();
-                                                                               cur_hop = Some(*next_id);
-                                                                       });
+                                                                       random_hop_offset = dir_info.direction().cltv_expiry_delta.into();
+                                                                       cur_hop = Some(*next_id);
                                                                }
                                                        });
                                                }
@@ -5214,14 +5249,12 @@ mod tests {
                                        for channel_id in &cur_node.channels {
                                                if let Some(channel_info) = network_channels.get(&channel_id) {
                                                        if let Some((dir_info, next_id)) = channel_info.as_directed_from(&cur_node_id) {
-                                                               if let Some(channel_update_info) = dir_info.direction() {
-                                                                       let next_cltv_expiry_delta = channel_update_info.cltv_expiry_delta as u32;
-                                                                       if cur_path_cltv_deltas.iter().sum::<u32>()
-                                                                               .saturating_add(next_cltv_expiry_delta) <= observed_cltv_expiry_delta {
-                                                                               let mut new_path_cltv_deltas = cur_path_cltv_deltas.clone();
-                                                                               new_path_cltv_deltas.push(next_cltv_expiry_delta);
-                                                                               candidates.push_back((*next_id, new_path_cltv_deltas));
-                                                                       }
+                                                               let next_cltv_expiry_delta = dir_info.direction().cltv_expiry_delta as u32;
+                                                               if cur_path_cltv_deltas.iter().sum::<u32>()
+                                                                       .saturating_add(next_cltv_expiry_delta) <= observed_cltv_expiry_delta {
+                                                                       let mut new_path_cltv_deltas = cur_path_cltv_deltas.clone();
+                                                                       new_path_cltv_deltas.push(next_cltv_expiry_delta);
+                                                                       candidates.push_back((*next_id, new_path_cltv_deltas));
                                                                }
                                                        }
                                                }
@@ -5398,7 +5431,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 0,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_000 },
                };
                scorer.set_manual_penalty(&NodeId::from_pubkey(&nodes[3]), 123);
                scorer.set_manual_penalty(&NodeId::from_pubkey(&nodes[4]), 456);
index 1c13771b1b7224432b1e79db5c2ceec6a97d752a..860f3cb195418054322cd551f5e7487b84af8cad 100644 (file)
@@ -1113,7 +1113,7 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, T: Time> Score for Probabilis
                                        return base_penalty_msat;
                                }
                        },
-                       EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: Some(htlc_maximum_msat) } => {
+                       EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat } => {
                                if htlc_maximum_msat >= capacity_msat/2 {
                                        anti_probing_penalty_msat = self.params.anti_probing_penalty_msat;
                                }
@@ -1143,31 +1143,32 @@ impl<G: Deref<Target = NetworkGraph<L>>, L: Deref, T: Time> Score for Probabilis
                                .get(&hop.short_channel_id)
                                .and_then(|channel| channel.as_directed_to(&target));
 
-                       if hop.short_channel_id == short_channel_id && hop_idx == 0 {
+                       let at_failed_channel = hop.short_channel_id == short_channel_id;
+                       if at_failed_channel && hop_idx == 0 {
                                log_warn!(self.logger, "Payment failed at the first hop - we do not attempt to learn channel info in such cases as we can directly observe local state.\n\tBecause we know the local state, we should generally not see failures here - this may be an indication that your channel peer on channel {} is broken and you may wish to close the channel.", hop.short_channel_id);
                        }
 
                        // Only score announced channels.
                        if let Some((channel, source)) = channel_directed_from_source {
                                let capacity_msat = channel.effective_capacity().as_msat();
-                               if hop.short_channel_id == short_channel_id {
+                               if at_failed_channel {
                                        self.channel_liquidities
                                                .entry(hop.short_channel_id)
                                                .or_insert_with(ChannelLiquidity::new)
                                                .as_directed_mut(source, &target, capacity_msat, &self.params)
                                                .failed_at_channel(amount_msat, format_args!("SCID {}, towards {:?}", hop.short_channel_id, target), &self.logger);
-                                       break;
+                               } else {
+                                       self.channel_liquidities
+                                               .entry(hop.short_channel_id)
+                                               .or_insert_with(ChannelLiquidity::new)
+                                               .as_directed_mut(source, &target, capacity_msat, &self.params)
+                                               .failed_downstream(amount_msat, format_args!("SCID {}, towards {:?}", hop.short_channel_id, target), &self.logger);
                                }
-
-                               self.channel_liquidities
-                                       .entry(hop.short_channel_id)
-                                       .or_insert_with(ChannelLiquidity::new)
-                                       .as_directed_mut(source, &target, capacity_msat, &self.params)
-                                       .failed_downstream(amount_msat, format_args!("SCID {}, towards {:?}", hop.short_channel_id, target), &self.logger);
                        } else {
                                log_debug!(self.logger, "Not able to penalize channel with SCID {} as we do not have graph info for it (likely a route-hint last-hop).",
                                        hop.short_channel_id);
                        }
+                       if at_failed_channel { break; }
                }
        }
 
@@ -1745,32 +1746,22 @@ mod tests {
                network_graph.update_channel(&signed_update).unwrap();
        }
 
+       fn path_hop(pubkey: PublicKey, short_channel_id: u64, fee_msat: u64) -> RouteHop {
+               RouteHop {
+                       pubkey,
+                       node_features: channelmanager::provided_node_features(),
+                       short_channel_id,
+                       channel_features: channelmanager::provided_channel_features(),
+                       fee_msat,
+                       cltv_expiry_delta: 18,
+               }
+       }
+
        fn payment_path_for_amount(amount_msat: u64) -> Vec<RouteHop> {
                vec![
-                       RouteHop {
-                               pubkey: source_pubkey(),
-                               node_features: channelmanager::provided_node_features(),
-                               short_channel_id: 41,
-                               channel_features: channelmanager::provided_channel_features(),
-                               fee_msat: 1,
-                               cltv_expiry_delta: 18,
-                       },
-                       RouteHop {
-                               pubkey: target_pubkey(),
-                               node_features: channelmanager::provided_node_features(),
-                               short_channel_id: 42,
-                               channel_features: channelmanager::provided_channel_features(),
-                               fee_msat: 2,
-                               cltv_expiry_delta: 18,
-                       },
-                       RouteHop {
-                               pubkey: recipient_pubkey(),
-                               node_features: channelmanager::provided_node_features(),
-                               short_channel_id: 43,
-                               channel_features: channelmanager::provided_channel_features(),
-                               fee_msat: amount_msat,
-                               cltv_expiry_delta: 18,
-                       },
+                       path_hop(source_pubkey(), 41, 1),
+                       path_hop(target_pubkey(), 42, 2),
+                       path_hop(recipient_pubkey(), 43, amount_msat),
                ]
        }
 
@@ -1985,7 +1976,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 1_024,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
                let usage = ChannelUsage { amount_msat: 10_240, ..usage };
@@ -1998,7 +1989,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 128,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 58);
                let usage = ChannelUsage { amount_msat: 256, ..usage };
@@ -2038,7 +2029,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 39,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 100, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 100, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
                let usage = ChannelUsage { amount_msat: 50, ..usage };
@@ -2062,7 +2053,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 500,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
                let failed_path = payment_path_for_amount(500);
                let successful_path = payment_path_for_amount(200);
@@ -2092,7 +2083,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 250,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 128);
                let usage = ChannelUsage { amount_msat: 500, ..usage };
@@ -2127,7 +2118,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 250,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 128);
                let usage = ChannelUsage { amount_msat: 500, ..usage };
@@ -2145,6 +2136,65 @@ mod tests {
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), u64::max_value());
        }
 
+       #[test]
+       fn ignores_channels_after_removed_failed_channel() {
+               // Previously, if we'd tried to send over a channel which was removed from the network
+               // graph before we call `payment_path_failed` (which is the default if the we get a "no
+               // such channel" error in the `InvoicePayer`), we would call `failed_downstream` on all
+               // channels in the route, even ones which they payment never reached. This tests to ensure
+               // we do not score such channels.
+               let secp_ctx = Secp256k1::new();
+               let logger = TestLogger::new();
+               let genesis_hash = genesis_block(Network::Testnet).header.block_hash();
+               let mut network_graph = NetworkGraph::new(genesis_hash, &logger);
+               let secret_a = SecretKey::from_slice(&[42; 32]).unwrap();
+               let secret_b = SecretKey::from_slice(&[43; 32]).unwrap();
+               let secret_c = SecretKey::from_slice(&[44; 32]).unwrap();
+               let secret_d = SecretKey::from_slice(&[45; 32]).unwrap();
+               add_channel(&mut network_graph, 42, secret_a, secret_b);
+               // Don't add the channel from B -> C.
+               add_channel(&mut network_graph, 44, secret_c, secret_d);
+
+               let pub_a = PublicKey::from_secret_key(&secp_ctx, &secret_a);
+               let pub_b = PublicKey::from_secret_key(&secp_ctx, &secret_b);
+               let pub_c = PublicKey::from_secret_key(&secp_ctx, &secret_c);
+               let pub_d = PublicKey::from_secret_key(&secp_ctx, &secret_d);
+
+               let path = vec![
+                       path_hop(pub_b, 42, 1),
+                       path_hop(pub_c, 43, 2),
+                       path_hop(pub_d, 44, 100),
+               ];
+
+               let node_a = NodeId::from_pubkey(&pub_a);
+               let node_b = NodeId::from_pubkey(&pub_b);
+               let node_c = NodeId::from_pubkey(&pub_c);
+               let node_d = NodeId::from_pubkey(&pub_d);
+
+               let params = ProbabilisticScoringParameters {
+                       liquidity_penalty_multiplier_msat: 1_000,
+                       ..ProbabilisticScoringParameters::zero_penalty()
+               };
+               let mut scorer = ProbabilisticScorer::new(params, &network_graph, &logger);
+
+               let usage = ChannelUsage {
+                       amount_msat: 250,
+                       inflight_htlc_msat: 0,
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
+               };
+               assert_eq!(scorer.channel_penalty_msat(42, &node_a, &node_b, usage), 128);
+               // Note that a default liquidity bound is used for B -> C as no channel exists
+               assert_eq!(scorer.channel_penalty_msat(43, &node_b, &node_c, usage), 128);
+               assert_eq!(scorer.channel_penalty_msat(44, &node_c, &node_d, usage), 128);
+
+               scorer.payment_path_failed(&path.iter().collect::<Vec<_>>(), 43);
+
+               assert_eq!(scorer.channel_penalty_msat(42, &node_a, &node_b, usage), 80);
+               // Note that a default liquidity bound is used for B -> C as no channel exists
+               assert_eq!(scorer.channel_penalty_msat(43, &node_b, &node_c, usage), 128);
+               assert_eq!(scorer.channel_penalty_msat(44, &node_c, &node_d, usage), 128);
+       }
+
        #[test]
        fn reduces_liquidity_upper_bound_along_path_on_success() {
                let logger = TestLogger::new();
@@ -2161,7 +2211,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 250,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
                let path = payment_path_for_amount(500);
 
@@ -2193,7 +2243,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 0,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_024) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_024 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
                let usage = ChannelUsage { amount_msat: 1_023, ..usage };
@@ -2271,7 +2321,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 256,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 125);
 
@@ -2302,7 +2352,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_000 },
                };
 
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 300);
@@ -2347,7 +2397,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 500,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
 
                scorer.payment_path_failed(&payment_path_for_amount(500).iter().collect::<Vec<_>>(), 42);
@@ -2384,7 +2434,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 500,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
 
                scorer.payment_path_failed(&payment_path_for_amount(500).iter().collect::<Vec<_>>(), 42);
@@ -2421,47 +2471,47 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 100_000_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 950_000_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 950_000_000, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 4375);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2739);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 2_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 2_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 2236);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 3_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 3_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1983);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 4_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 4_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1637);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 5_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 5_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1606);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 6_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 6_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1331);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_450_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_450_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1387);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 7_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1379);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 8_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 8_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1363);
                let usage = ChannelUsage {
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 9_950_000_000, htlc_maximum_msat: Some(1_000) }, ..usage
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 9_950_000_000, htlc_maximum_msat: 1_000 }, ..usage
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 1355);
        }
@@ -2475,7 +2525,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 128,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_000 },
                };
 
                let params = ProbabilisticScoringParameters {
@@ -2511,7 +2561,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_000 },
                };
 
                let params = ProbabilisticScoringParameters {
@@ -2566,7 +2616,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 750,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_000, htlc_maximum_msat: 1_000 },
                };
                assert_ne!(scorer.channel_penalty_msat(42, &source, &target, usage), u64::max_value());
 
@@ -2615,7 +2665,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 100,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: Some(1_024) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024, htlc_maximum_msat: 1_024 },
                };
                // With no historical data the normal liquidity penalty calculation is used.
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 47);
@@ -2650,7 +2700,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
 
@@ -2658,7 +2708,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_024_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_024_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500);
 
@@ -2666,7 +2716,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(512_000) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 512_000 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 500);
 
@@ -2674,7 +2724,7 @@ mod tests {
                let usage = ChannelUsage {
                        amount_msat: 512_000,
                        inflight_htlc_msat: 0,
-                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(511_999) },
+                       effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 511_999 },
                };
                assert_eq!(scorer.channel_penalty_msat(42, &source, &target, usage), 0);
        }
index 1b75648019ae52f916f303b92017508d429a4c08..552226ca3e64121569cd2ff65589152255db9c05 100644 (file)
@@ -228,7 +228,6 @@ impl<'a, T: Writeable> Writeable for ChaChaPolyWriteAdapter<'a, T> {
 /// Enables the use of the serialization macros for objects that need to be simultaneously decrypted and
 /// deserialized. This allows us to avoid an intermediate Vec allocation.
 pub(crate) struct ChaChaPolyReadAdapter<R: Readable> {
-       #[allow(unused)] // This will be used soon for onion messages
        pub readable: R,
 }
 
index 8344f9f357f959b5d62a4e35ee4e1a677ddfa8b4..6317b43d5ed3b406f13f2f0f932bf59c6b07947c 100644 (file)
@@ -201,7 +201,7 @@ impl BaseSign for EnforcingSigner {
        }
 
        fn sign_holder_anchor_input(
-               &self, anchor_tx: &mut Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
+               &self, anchor_tx: &Transaction, input: usize, secp_ctx: &Secp256k1<secp256k1::All>,
        ) -> Result<Signature, ()> {
                debug_assert!(MIN_CHAN_DUST_LIMIT_SATOSHIS > ANCHOR_OUTPUT_VALUE_SATOSHI);
                // As long as our minimum dust limit is enforced and is greater than our anchor output
index 37a146c3dacb16420f5c3c2138ea3629d7fc6833..9818bfc4c8da0eb8fc746a8133d08df84aaaa5d8 100644 (file)
@@ -15,6 +15,7 @@
 //! few other things.
 
 use crate::chain::keysinterface::SpendableOutputDescriptor;
+#[cfg(anchors)]
 use crate::ln::chan_utils::HTLCOutputInCommitment;
 use crate::ln::channelmanager::PaymentId;
 use crate::ln::channel::FUNDING_CONF_DEADLINE_BLOCKS;
@@ -23,10 +24,12 @@ use crate::ln::msgs;
 use crate::ln::msgs::DecodeError;
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 use crate::routing::gossip::NetworkUpdate;
-use crate::util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, VecReadWrapper, VecWriteWrapper};
+use crate::util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, WithoutLength, OptionDeserWrapper};
 use crate::routing::router::{RouteHop, RouteParameters};
 
-use bitcoin::{PackedLockTime, Transaction, OutPoint};
+use bitcoin::{PackedLockTime, Transaction};
+#[cfg(anchors)]
+use bitcoin::OutPoint;
 use bitcoin::blockdata::script::Script;
 use bitcoin::hashes::Hash;
 use bitcoin::hashes::sha256::Hash as Sha256;
@@ -406,8 +409,8 @@ pub enum Event {
        /// provide failure information for each MPP part in the payment.
        ///
        /// This event is provided once there are no further pending HTLCs for the payment and the
-       /// payment is no longer retryable, either due to a several-block timeout or because
-       /// [`ChannelManager::abandon_payment`] was previously called for the corresponding payment.
+       /// payment is no longer retryable due to [`ChannelManager::abandon_payment`] having been
+       /// called for the corresponding payment.
        ///
        /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
        PaymentFailed {
@@ -446,9 +449,14 @@ pub enum Event {
        /// Indicates an outbound HTLC we sent failed. Probably some intermediary node dropped
        /// something. You may wish to retry with a different route.
        ///
+       /// If you have given up retrying this payment and wish to fail it, you MUST call
+       /// [`ChannelManager::abandon_payment`] at least once for a given [`PaymentId`] or memory
+       /// related to payment tracking will leak.
+       ///
        /// Note that this does *not* indicate that all paths for an MPP payment have failed, see
        /// [`Event::PaymentFailed`] and [`all_paths_failed`].
        ///
+       /// [`ChannelManager::abandon_payment`]: crate::ln::channelmanager::ChannelManager::abandon_payment
        /// [`all_paths_failed`]: Self::PaymentPathFailed::all_paths_failed
        PaymentPathFailed {
                /// The id returned by [`ChannelManager::send_payment`] and used with
@@ -594,6 +602,27 @@ pub enum Event {
                /// transaction.
                claim_from_onchain_tx: bool,
        },
+       /// Used to indicate that a channel with the given `channel_id` is ready to
+       /// be used. This event is emitted either when the funding transaction has been confirmed
+       /// on-chain, or, in case of a 0conf channel, when both parties have confirmed the channel
+       /// establishment.
+       ChannelReady {
+               /// The channel_id of the channel that is ready.
+               channel_id: [u8; 32],
+               /// The `user_channel_id` value passed in to [`ChannelManager::create_channel`] for outbound
+               /// channels, or to [`ChannelManager::accept_inbound_channel`] for inbound channels if
+               /// [`UserConfig::manually_accept_inbound_channels`] config flag is set to true. Otherwise
+               /// `user_channel_id` will be 0 for an inbound channel.
+               ///
+               /// [`ChannelManager::create_channel`]: crate::ln::channelmanager::ChannelManager::create_channel
+               /// [`ChannelManager::accept_inbound_channel`]: crate::ln::channelmanager::ChannelManager::accept_inbound_channel
+               /// [`UserConfig::manually_accept_inbound_channels`]: crate::util::config::UserConfig::manually_accept_inbound_channels
+               user_channel_id: u64,
+               /// The node_id of the channel counterparty.
+               counterparty_node_id: PublicKey,
+               /// The features that this channel will operate with.
+               channel_type: ChannelTypeFeatures,
+       },
        /// Used to indicate that a previously opened channel with the given `channel_id` is in the
        /// process of closure.
        ChannelClosed  {
@@ -756,7 +785,7 @@ impl Writeable for Event {
                                        (1, network_update, option),
                                        (2, payment_failed_permanently, required),
                                        (3, all_paths_failed, required),
-                                       (5, path, vec_type),
+                                       (5, *path, vec_type),
                                        (7, short_channel_id, option),
                                        (9, retry, option),
                                        (11, payment_id, option),
@@ -770,7 +799,7 @@ impl Writeable for Event {
                        &Event::SpendableOutputs { ref outputs } => {
                                5u8.write(writer)?;
                                write_tlv_fields!(writer, {
-                                       (0, VecWriteWrapper(outputs), required),
+                                       (0, WithoutLength(outputs), required),
                                });
                        },
                        &Event::PaymentForwarded { fee_earned_msat, prev_channel_id, claim_from_onchain_tx, next_channel_id } => {
@@ -802,7 +831,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, option),
-                                       (4, path, vec_type)
+                                       (4, *path, vec_type)
                                })
                        },
                        &Event::PaymentFailed { ref payment_id, ref payment_hash } => {
@@ -830,7 +859,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, path, vec_type)
+                                       (4, *path, vec_type)
                                })
                        },
                        &Event::ProbeFailed { ref payment_id, ref payment_hash, ref path, ref short_channel_id } => {
@@ -838,7 +867,7 @@ impl Writeable for Event {
                                write_tlv_fields!(writer, {
                                        (0, payment_id, required),
                                        (2, payment_hash, required),
-                                       (4, path, vec_type),
+                                       (4, *path, vec_type),
                                        (6, short_channel_id, option),
                                })
                        },
@@ -858,6 +887,15 @@ impl Writeable for Event {
                                        BumpTransactionEvent::ChannelClose { .. } => {}
                                }
                        }
+                       &Event::ChannelReady { ref channel_id, ref user_channel_id, ref counterparty_node_id, ref channel_type } => {
+                               29u8.write(writer)?;
+                               write_tlv_fields!(writer, {
+                                       (0, channel_id, required),
+                                       (2, user_channel_id, required),
+                                       (4, counterparty_node_id, required),
+                                       (6, channel_type, required),
+                               });
+                       },
                        // Note that, going forward, all new events must only write data inside of
                        // `write_tlv_fields`. Versions 0.0.101+ will ignore odd-numbered events that write
                        // data via `write_tlv_fields`.
@@ -969,7 +1007,7 @@ impl MaybeReadable for Event {
                        4u8 => Ok(None),
                        5u8 => {
                                let f = || {
-                                       let mut outputs = VecReadWrapper(Vec::new());
+                                       let mut outputs = WithoutLength(Vec::new());
                                        read_tlv_fields!(reader, {
                                                (0, outputs, required),
                                        });
@@ -1138,6 +1176,29 @@ impl MaybeReadable for Event {
                                };
                                f()
                        },
+                       27u8 => Ok(None),
+                       29u8 => {
+                               let f = || {
+                                       let mut channel_id = [0; 32];
+                                       let mut user_channel_id: u64 = 0;
+                                       let mut counterparty_node_id = OptionDeserWrapper(None);
+                                       let mut channel_type = OptionDeserWrapper(None);
+                                       read_tlv_fields!(reader, {
+                                               (0, channel_id, required),
+                                               (2, user_channel_id, required),
+                                               (4, counterparty_node_id, required),
+                                               (6, channel_type, required),
+                                       });
+
+                                       Ok(Some(Event::ChannelReady {
+                                               channel_id,
+                                               user_channel_id,
+                                               counterparty_node_id: counterparty_node_id.0.unwrap(),
+                                               channel_type: channel_type.0.unwrap()
+                                       }))
+                               };
+                               f()
+                       },
                        // Versions prior to 0.0.100 did not ignore odd types, instead returning InvalidValue.
                        // Version 0.0.100 failed to properly ignore odd types, possibly resulting in corrupt
                        // reads.
@@ -1339,6 +1400,10 @@ pub trait OnionMessageProvider {
 ///
 /// Events are processed by passing an [`EventHandler`] to [`process_pending_events`].
 ///
+/// Implementations of this trait may also feature an async version of event handling, as shown with
+/// [`ChannelManager::process_pending_events_async`] and
+/// [`ChainMonitor::process_pending_events_async`].
+///
 /// # Requirements
 ///
 /// When using this trait, [`process_pending_events`] will call [`handle_event`] for each pending
@@ -1365,6 +1430,8 @@ pub trait OnionMessageProvider {
 /// [`handle_event`]: EventHandler::handle_event
 /// [`ChannelManager::process_pending_events`]: crate::ln::channelmanager::ChannelManager#method.process_pending_events
 /// [`ChainMonitor::process_pending_events`]: crate::chain::chainmonitor::ChainMonitor#method.process_pending_events
+/// [`ChannelManager::process_pending_events_async`]: crate::ln::channelmanager::ChannelManager::process_pending_events_async
+/// [`ChainMonitor::process_pending_events_async`]: crate::chain::chainmonitor::ChainMonitor::process_pending_events_async
 pub trait EventsProvider {
        /// Processes any events generated since the last call using the given event handler.
        ///
@@ -1373,21 +1440,25 @@ pub trait EventsProvider {
 }
 
 /// A trait implemented for objects handling events from [`EventsProvider`].
+///
+/// An async variation also exists for implementations of [`EventsProvider`] that support async
+/// event handling. The async event handler should satisfy the generic bounds: `F:
+/// core::future::Future, H: Fn(Event) -> F`.
 pub trait EventHandler {
        /// Handles the given [`Event`].
        ///
        /// See [`EventsProvider`] for details that must be considered when implementing this method.
-       fn handle_event(&self, event: &Event);
+       fn handle_event(&self, event: Event);
 }
 
-impl<F> EventHandler for F where F: Fn(&Event) {
-       fn handle_event(&self, event: &Event) {
+impl<F> EventHandler for F where F: Fn(Event) {
+       fn handle_event(&self, event: Event) {
                self(event)
        }
 }
 
 impl<T: EventHandler> EventHandler for Arc<T> {
-       fn handle_event(&self, event: &Event) {
+       fn handle_event(&self, event: Event) {
                self.deref().handle_event(event)
        }
 }
index 9ffe1b7494fae459bdbab3e26a593c35678ee421..730131f224197dc38e5719be8db2336b92089eae 100644 (file)
@@ -21,6 +21,7 @@ pub mod ser;
 pub mod message_signing;
 pub mod invoice;
 pub mod persist;
+pub mod string;
 pub mod wakers;
 
 pub(crate) mod atomic_counter;
index 9e81a3406cdc3acb0e007676f82adb0901ffa2c4..40c042a27ff702c457a1c856db2097720cd4caf3 100644 (file)
@@ -26,15 +26,15 @@ pub trait KVStorePersister {
 }
 
 /// Trait that handles persisting a [`ChannelManager`], [`NetworkGraph`], and [`WriteableScore`] to disk.
-pub trait Persister<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref, S: WriteableScore<'a>>
-       where M::Target: 'static + chain::Watch<Signer>,
+pub trait Persister<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref, S: WriteableScore<'a>>
+       where M::Target: 'static + chain::Watch<<K::Target as KeysInterface>::Signer>,
                T::Target: 'static + BroadcasterInterface,
-               K::Target: 'static + KeysInterface<Signer = Signer>,
+               K::Target: 'static + KeysInterface,
                F::Target: 'static + FeeEstimator,
                L::Target: 'static + Logger,
 {
        /// Persist the given ['ChannelManager'] to disk, returning an error if persistence failed.
-       fn persist_manager(&self, channel_manager: &ChannelManager<Signer, M, T, K, F, L>) -> Result<(), io::Error>;
+       fn persist_manager(&self, channel_manager: &ChannelManager<M, T, K, F, L>) -> Result<(), io::Error>;
 
        /// Persist the given [`NetworkGraph`] to disk, returning an error if persistence failed.
        fn persist_graph(&self, network_graph: &NetworkGraph<L>) -> Result<(), io::Error>;
@@ -43,15 +43,15 @@ pub trait Persister<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L:
        fn persist_scorer(&self, scorer: &S) -> Result<(), io::Error>;
 }
 
-impl<'a, A: KVStorePersister, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref, S: WriteableScore<'a>> Persister<'a, Signer, M, T, K, F, L, S> for A
-       where M::Target: 'static + chain::Watch<Signer>,
+impl<'a, A: KVStorePersister, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref, S: WriteableScore<'a>> Persister<'a, M, T, K, F, L, S> for A
+       where M::Target: 'static + chain::Watch<<K::Target as KeysInterface>::Signer>,
                T::Target: 'static + BroadcasterInterface,
-               K::Target: 'static + KeysInterface<Signer = Signer>,
+               K::Target: 'static + KeysInterface,
                F::Target: 'static + FeeEstimator,
                L::Target: 'static + Logger,
 {
        /// Persist the given ['ChannelManager'] to disk with the name "manager", returning an error if persistence failed.
-       fn persist_manager(&self, channel_manager: &ChannelManager<Signer, M, T, K, F, L>) -> Result<(), io::Error> {
+       fn persist_manager(&self, channel_manager: &ChannelManager<M, T, K, F, L>) -> Result<(), io::Error> {
                self.persist("manager", channel_manager)
        }
 
index b851cca81aa06150327bcd5b1c45cc2238c54fd9..651b36ef32c3ce5a7d4382be73562733a5fb8a4b 100644 (file)
@@ -66,14 +66,14 @@ pub fn scid_from_parts(block: u64, tx_index: u64, vout_index: u64) -> Result<u64
 pub(crate) mod fake_scid {
        use bitcoin::hash_types::BlockHash;
        use bitcoin::hashes::hex::FromHex;
-       use crate::chain::keysinterface::{Sign, KeysInterface};
+       use crate::chain::keysinterface::KeysInterface;
        use crate::util::chacha20::ChaCha20;
        use crate::util::scid_utils;
 
        use core::convert::TryInto;
        use core::ops::Deref;
 
-       const TEST_SEGWIT_ACTIVATION_HEIGHT: u32 = 0;
+       const TEST_SEGWIT_ACTIVATION_HEIGHT: u32 = 1;
        const MAINNET_SEGWIT_ACTIVATION_HEIGHT: u32 = 481_824;
        const MAX_TX_INDEX: u32 = 2_500;
        const MAX_NAMESPACES: u8 = 8; // We allocate 3 bits for the namespace identifier.
@@ -98,8 +98,8 @@ pub(crate) mod fake_scid {
                /// between segwit activation and the current best known height, and the tx index and output
                /// index are also selected from a "reasonable" range. We add this logic because it makes it
                /// non-obvious at a glance that the scid is fake, e.g. if it appears in invoice route hints.
-               pub(crate) fn get_fake_scid<Signer: Sign, K: Deref>(&self, highest_seen_blockheight: u32, genesis_hash: &BlockHash, fake_scid_rand_bytes: &[u8; 32], keys_manager: &K) -> u64
-                       where K::Target: KeysInterface<Signer = Signer>,
+               pub(crate) fn get_fake_scid<K: Deref>(&self, highest_seen_blockheight: u32, genesis_hash: &BlockHash, fake_scid_rand_bytes: &[u8; 32], keys_manager: &K) -> u64
+                       where K::Target: KeysInterface,
                {
                        // Ensure we haven't created a namespace that doesn't fit into the 3 bits we've allocated for
                        // namespaces.
@@ -151,12 +151,13 @@ pub(crate) mod fake_scid {
        }
 
        /// Returns whether the given fake scid falls into the given namespace.
-       pub fn is_valid_phantom(fake_scid_rand_bytes: &[u8; 32], scid: u64) -> bool {
+       pub fn is_valid_phantom(fake_scid_rand_bytes: &[u8; 32], scid: u64, genesis_hash: &BlockHash) -> bool {
                let block_height = scid_utils::block_from_scid(&scid);
                let tx_index = scid_utils::tx_index_from_scid(&scid);
                let namespace = Namespace::Phantom;
                let valid_vout = namespace.get_encrypted_vout(block_height, tx_index, fake_scid_rand_bytes);
-               valid_vout == scid_utils::vout_from_scid(&scid) as u8
+               block_height >= segwit_activation_height(genesis_hash)
+                       && valid_vout == scid_utils::vout_from_scid(&scid) as u8
        }
 
        #[cfg(test)]
@@ -194,11 +195,12 @@ pub(crate) mod fake_scid {
                fn test_is_valid_phantom() {
                        let namespace = Namespace::Phantom;
                        let fake_scid_rand_bytes = [0; 32];
+                       let testnet_genesis = genesis_block(Network::Testnet).header.block_hash();
                        let valid_encrypted_vout = namespace.get_encrypted_vout(0, 0, &fake_scid_rand_bytes);
-                       let valid_fake_scid = scid_utils::scid_from_parts(0, 0, valid_encrypted_vout as u64).unwrap();
-                       assert!(is_valid_phantom(&fake_scid_rand_bytes, valid_fake_scid));
-                       let invalid_fake_scid = scid_utils::scid_from_parts(0, 0, 12).unwrap();
-                       assert!(!is_valid_phantom(&fake_scid_rand_bytes, invalid_fake_scid));
+                       let valid_fake_scid = scid_utils::scid_from_parts(1, 0, valid_encrypted_vout as u64).unwrap();
+                       assert!(is_valid_phantom(&fake_scid_rand_bytes, valid_fake_scid, &testnet_genesis));
+                       let invalid_fake_scid = scid_utils::scid_from_parts(1, 0, 12).unwrap();
+                       assert!(!is_valid_phantom(&fake_scid_rand_bytes, invalid_fake_scid, &testnet_genesis));
                }
 
                #[test]
index 1bf30fa9f72e421882b1fe612da08c30b65a29eb..7c4ba7fd50f99888745a6944971c10f3f00f77a9 100644 (file)
@@ -22,6 +22,7 @@ use core::ops::Deref;
 use bitcoin::secp256k1::{PublicKey, SecretKey};
 use bitcoin::secp256k1::constants::{PUBLIC_KEY_SIZE, SECRET_KEY_SIZE, COMPACT_SIGNATURE_SIZE};
 use bitcoin::secp256k1::ecdsa::Signature;
+use bitcoin::blockdata::constants::ChainHash;
 use bitcoin::blockdata::script::Script;
 use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
 use bitcoin::consensus;
@@ -269,15 +270,6 @@ impl<T: Readable> MaybeReadable for T {
        }
 }
 
-/// A trait that various rust-lightning types implement allowing them to (maybe) be read in from a
-/// Read, given some additional set of arguments which is required to deserialize.
-///
-/// (C-not exported) as we only export serialization to/from byte arrays instead
-pub trait MaybeReadableArgs<P> {
-       /// Reads a Self in from the given Read
-       fn read<R: Read>(reader: &mut R, params: P) -> Result<Option<Self>, DecodeError> where Self: Sized;
-}
-
 pub(crate) struct OptionDeserWrapper<T: Readable>(pub Option<T>);
 impl<T: Readable> Readable for OptionDeserWrapper<T> {
        #[inline]
@@ -292,39 +284,6 @@ impl<T: Readable> From<T> for OptionDeserWrapper<T> {
        fn from(t: T) -> OptionDeserWrapper<T> { OptionDeserWrapper(Some(t)) }
 }
 
-/// Wrapper to write each element of a Vec with no length prefix
-pub(crate) struct VecWriteWrapper<'a, T: Writeable>(pub &'a Vec<T>);
-impl<'a, T: Writeable> Writeable for VecWriteWrapper<'a, T> {
-       #[inline]
-       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
-               for ref v in self.0.iter() {
-                       v.write(writer)?;
-               }
-               Ok(())
-       }
-}
-
-/// Wrapper to read elements from a given stream until it reaches the end of the stream.
-pub(crate) struct VecReadWrapper<T>(pub Vec<T>);
-impl<T: MaybeReadable> Readable for VecReadWrapper<T> {
-       #[inline]
-       fn read<R: Read>(mut reader: &mut R) -> Result<Self, DecodeError> {
-               let mut values = Vec::new();
-               loop {
-                       let mut track_read = ReadTrackingReader::new(&mut reader);
-                       match MaybeReadable::read(&mut track_read) {
-                               Ok(Some(v)) => { values.push(v); },
-                               Ok(None) => { },
-                               // If we failed to read any bytes at all, we reached the end of our TLV
-                               // stream and have simply exhausted all entries.
-                               Err(ref e) if e == &DecodeError::ShortRead && !track_read.have_read => break,
-                               Err(e) => return Err(e),
-                       }
-               }
-               Ok(Self(values))
-       }
-}
-
 pub(crate) struct U48(pub u64);
 impl Writeable for U48 {
        #[inline]
@@ -460,6 +419,9 @@ macro_rules! impl_writeable_primitive {
                                }
                        }
                }
+               impl From<$val_type> for HighZeroBytesDroppedBigSize<$val_type> {
+                       fn from(val: $val_type) -> Self { Self(val) }
+               }
        }
 }
 
@@ -523,7 +485,7 @@ macro_rules! impl_array {
        );
 }
 
-impl_array!(3); // for rgb
+impl_array!(3); // for rgb, ISO 4712 code
 impl_array!(4); // for IPv4
 impl_array!(12); // for OnionV2
 impl_array!(16); // for IPv6
@@ -555,6 +517,59 @@ impl Readable for [u16; 8] {
        }
 }
 
+/// For variable-length values within TLV record where the length is encoded as part of the record.
+/// Used to prevent encoding the length twice.
+pub(crate) struct WithoutLength<T>(pub T);
+
+impl Writeable for WithoutLength<&String> {
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               w.write_all(self.0.as_bytes())
+       }
+}
+impl Readable for WithoutLength<String> {
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let v: WithoutLength<Vec<u8>> = Readable::read(r)?;
+               Ok(Self(String::from_utf8(v.0).map_err(|_| DecodeError::InvalidValue)?))
+       }
+}
+impl<'a> From<&'a String> for WithoutLength<&'a String> {
+       fn from(s: &'a String) -> Self { Self(s) }
+}
+
+impl<'a, T: Writeable> Writeable for WithoutLength<&'a Vec<T>> {
+       #[inline]
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               for ref v in self.0.iter() {
+                       v.write(writer)?;
+               }
+               Ok(())
+       }
+}
+
+impl<T: MaybeReadable> Readable for WithoutLength<Vec<T>> {
+       #[inline]
+       fn read<R: Read>(mut reader: &mut R) -> Result<Self, DecodeError> {
+               let mut values = Vec::new();
+               loop {
+                       let mut track_read = ReadTrackingReader::new(&mut reader);
+                       match MaybeReadable::read(&mut track_read) {
+                               Ok(Some(v)) => { values.push(v); },
+                               Ok(None) => { },
+                               // If we failed to read any bytes at all, we reached the end of our TLV
+                               // stream and have simply exhausted all entries.
+                               Err(ref e) if e == &DecodeError::ShortRead && !track_read.have_read => break,
+                               Err(e) => return Err(e),
+                       }
+               }
+               Ok(Self(values))
+       }
+}
+impl<'a, T> From<&'a Vec<T>> for WithoutLength<&'a Vec<T>> {
+       fn from(v: &'a Vec<T>) -> Self { Self(v) }
+}
+
 // HashMap
 impl<K, V> Writeable for HashMap<K, V>
        where K: Writeable + Eq + Hash,
@@ -869,6 +884,19 @@ impl Readable for BlockHash {
        }
 }
 
+impl Writeable for ChainHash {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               w.write_all(self.as_bytes())
+       }
+}
+
+impl Readable for ChainHash {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let buf: [u8; 32] = Readable::read(r)?;
+               Ok(ChainHash::from(&buf[..]))
+       }
+}
+
 impl Writeable for OutPoint {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                self.txid.write(w)?;
index 74b206d3a5774ca8885a7693f2562c05187348be..3ec0848680ffe62a2532e03d47d1fc5b62d7f4b8 100644 (file)
@@ -17,7 +17,7 @@ macro_rules! encode_tlv {
                $field.write($stream)?;
        };
        ($stream: expr, $type: expr, $field: expr, vec_type) => {
-               encode_tlv!($stream, $type, $crate::util::ser::VecWriteWrapper(&$field), required);
+               encode_tlv!($stream, $type, $crate::util::ser::WithoutLength(&$field), required);
        };
        ($stream: expr, $optional_type: expr, $optional_field: expr, option) => {
                if let Some(ref field) = $optional_field {
@@ -26,6 +26,12 @@ macro_rules! encode_tlv {
                        field.write($stream)?;
                }
        };
+       ($stream: expr, $type: expr, $field: expr, (option, encoding: ($fieldty: ty, $encoding: ident))) => {
+               encode_tlv!($stream, $type, $field.map(|f| $encoding(f)), option);
+       };
+       ($stream: expr, $type: expr, $field: expr, (option, encoding: $fieldty: ty)) => {
+               encode_tlv!($stream, $type, $field, option);
+       };
 }
 
 macro_rules! encode_tlv_stream {
@@ -66,7 +72,7 @@ macro_rules! get_varint_length_prefixed_tlv_length {
                $len.0 += field_len;
        };
        ($len: expr, $type: expr, $field: expr, vec_type) => {
-               get_varint_length_prefixed_tlv_length!($len, $type, $crate::util::ser::VecWriteWrapper(&$field), required);
+               get_varint_length_prefixed_tlv_length!($len, $type, $crate::util::ser::WithoutLength(&$field), required);
        };
        ($len: expr, $optional_type: expr, $optional_field: expr, option) => {
                if let Some(ref field) = $optional_field {
@@ -121,6 +127,9 @@ macro_rules! check_tlv_order {
        ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $typ: expr, $type: expr, $field: ident, (option, encoding: $encoding: tt)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! check_missing_tlv {
@@ -150,6 +159,9 @@ macro_rules! check_missing_tlv {
        ($last_seen_type: expr, $type: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                // no-op
        }};
+       ($last_seen_type: expr, $type: expr, $field: ident, (option, encoding: $encoding: tt)) => {{
+               // no-op
+       }};
 }
 
 macro_rules! decode_tlv {
@@ -160,7 +172,7 @@ macro_rules! decode_tlv {
                $field = $crate::util::ser::Readable::read(&mut $reader)?;
        }};
        ($reader: expr, $field: ident, vec_type) => {{
-               let f: $crate::util::ser::VecReadWrapper<_> = $crate::util::ser::Readable::read(&mut $reader)?;
+               let f: $crate::util::ser::WithoutLength<Vec<_>> = $crate::util::ser::Readable::read(&mut $reader)?;
                $field = Some(f.0);
        }};
        ($reader: expr, $field: ident, option) => {{
@@ -172,6 +184,15 @@ macro_rules! decode_tlv {
        ($reader: expr, $field: ident, (option: $trait: ident $(, $read_arg: expr)?)) => {{
                $field = Some($trait::read(&mut $reader $(, $read_arg)*)?);
        }};
+       ($reader: expr, $field: ident, (option, encoding: ($fieldty: ty, $encoding: ident))) => {{
+               $field = {
+                       let field: $encoding<$fieldty> = ser::Readable::read(&mut $reader)?;
+                       Some(field.0)
+               };
+       }};
+       ($reader: expr, $field: ident, (option, encoding: $fieldty: ty)) => {{
+               decode_tlv!($reader, $field, option);
+       }};
 }
 
 // `$decode_custom_tlv` is a closure that may be optionally provided to handle custom message types.
@@ -441,6 +462,75 @@ macro_rules! impl_writeable_tlv_based {
        }
 }
 
+/// Defines a struct for a TLV stream and a similar struct using references for non-primitive types,
+/// implementing [`Readable`] for the former and [`Writeable`] for the latter. Useful as an
+/// intermediary format when reading or writing a type encoded as a TLV stream. Note that each field
+/// representing a TLV record has its type wrapped with an [`Option`]. A tuple consisting of a type
+/// and a serialization wrapper may be given in place of a type when custom serialization is
+/// required.
+///
+/// [`Readable`]: crate::util::ser::Readable
+/// [`Writeable`]: crate::util::ser::Writeable
+macro_rules! tlv_stream {
+       ($name:ident, $nameref:ident, {
+               $(($type:expr, $field:ident : $fieldty:tt)),* $(,)*
+       }) => {
+               #[derive(Debug)]
+               struct $name {
+                       $(
+                               $field: Option<tlv_record_type!($fieldty)>,
+                       )*
+               }
+
+               pub(crate) struct $nameref<'a> {
+                       $(
+                               pub(crate) $field: Option<tlv_record_ref_type!($fieldty)>,
+                       )*
+               }
+
+               impl<'a> $crate::util::ser::Writeable for $nameref<'a> {
+                       fn write<W: $crate::util::ser::Writer>(&self, writer: &mut W) -> Result<(), $crate::io::Error> {
+                               encode_tlv_stream!(writer, {
+                                       $(($type, self.$field, (option, encoding: $fieldty))),*
+                               });
+                               Ok(())
+                       }
+               }
+
+               impl $crate::util::ser::Readable for $name {
+                       fn read<R: $crate::io::Read>(reader: &mut R) -> Result<Self, $crate::ln::msgs::DecodeError> {
+                               $(
+                                       init_tlv_field_var!($field, option);
+                               )*
+                               decode_tlv_stream!(reader, {
+                                       $(($type, $field, (option, encoding: $fieldty))),*
+                               });
+
+                               Ok(Self {
+                                       $(
+                                               $field: $field
+                                       ),*
+                               })
+                       }
+               }
+       }
+}
+
+macro_rules! tlv_record_type {
+       (($type:ty, $wrapper:ident)) => { $type };
+       ($type:ty) => { $type };
+}
+
+macro_rules! tlv_record_ref_type {
+       (char) => { char };
+       (u8) => { u8 };
+       ((u16, $wrapper: ident)) => { u16 };
+       ((u32, $wrapper: ident)) => { u32 };
+       ((u64, $wrapper: ident)) => { u64 };
+       (($type:ty, $wrapper:ident)) => { &'a $type };
+       ($type:ty) => { &'a $type };
+}
+
 macro_rules! _impl_writeable_tlv_based_enum_common {
        ($st: ident, $(($variant_id: expr, $variant_name: ident) =>
                {$(($type: expr, $field: ident, $fieldty: tt)),* $(,)*}
@@ -453,7 +543,7 @@ macro_rules! _impl_writeable_tlv_based_enum_common {
                                                let id: u8 = $variant_id;
                                                id.write(writer)?;
                                                write_tlv_fields!(writer, {
-                                                       $(($type, $field, $fieldty)),*
+                                                       $(($type, *$field, $fieldty)),*
                                                });
                                        }),*
                                        $($st::$tuple_variant_name (ref field) => {
diff --git a/lightning/src/util/string.rs b/lightning/src/util/string.rs
new file mode 100644 (file)
index 0000000..42eb96f
--- /dev/null
@@ -0,0 +1,42 @@
+// This file is Copyright its original authors, visible in version control
+// history.
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Utilities for strings.
+
+use core::fmt;
+
+/// A string that displays only printable characters, replacing control characters with
+/// [`core::char::REPLACEMENT_CHARACTER`].
+#[derive(Debug, PartialEq)]
+pub struct PrintableString<'a>(pub &'a str);
+
+impl<'a> fmt::Display for PrintableString<'a> {
+       fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+               use core::fmt::Write;
+               for c in self.0.chars() {
+                       let c = if c.is_control() { core::char::REPLACEMENT_CHARACTER } else { c };
+                       f.write_char(c)?;
+               }
+
+               Ok(())
+       }
+}
+
+#[cfg(test)]
+mod tests {
+       use super::PrintableString;
+
+       #[test]
+       fn displays_printable_string() {
+               assert_eq!(
+                       format!("{}", PrintableString("I \u{1F496} LDK!\t\u{26A1}")),
+                       "I \u{1F496} LDK!\u{FFFD}\u{26A1}",
+               );
+       }
+}
diff --git a/pending_changelog/1743.txt b/pending_changelog/1743.txt
new file mode 100644 (file)
index 0000000..93e4120
--- /dev/null
@@ -0,0 +1,7 @@
+## API Updates
+- A new `ChannelReady` event is generated whenever a channel becomes ready to
+  be used, i.e., after both sides sent the `channel_ready` message.
+
+## Backwards Compatibilty
+- No `ChannelReady` events will be generated for previously existing channels, including
+  those which become ready after upgrading 0.0.113.
diff --git a/pending_changelog/matt-idempotent-payments.txt b/pending_changelog/matt-idempotent-payments.txt
new file mode 100644 (file)
index 0000000..e837594
--- /dev/null
@@ -0,0 +1,16 @@
+API Changes
+===========
+
+ * Payment sending methods now take an explicit `PaymentId`, which acts as an
+   idempotency token. You may use the PaymentHash for this, which existing
+   `InvoicePayer` send methods do, new `_with_id` variants were added (#XXXX).
+ * Pending outbound payments are no longer automatically timed-out a few blocks
+   after failure. Thus, in order to avoid leaking memory, you MUST call
+   `ChannelManager::abandon_payment` when you no longer wish to retry (#XXXX).
+
+Serialization Compatibility
+===========================
+
+ * When downgrading to a version of LDK prior to THIS_VERSION_XXX when there are
+   resolved payments waiting for a small timeout, the payments may not be
+   removed, preventing payments with the same `PaymentId`.