Merge pull request #1413 from ViktorTigerstrom/2022-04-default-to-bolt4-tlv-onions
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Fri, 11 Nov 2022 00:49:45 +0000 (00:49 +0000)
committerGitHub <noreply@github.com>
Fri, 11 Nov 2022 00:49:45 +0000 (00:49 +0000)
Drop support for creating BOLT 4 Legacy onion format payloads

1  2 
fuzz/src/full_stack.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/msgs.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/routing/router.rs

diff --combined fuzz/src/full_stack.rs
index 5e26a6fd0d8003b883edf774bca400d9b6297a43,5d9608a7c790248b5171e45fd3a33093b988ad4a..eeb3728adf59dc789653a12e4c54835a5a1ea979
@@@ -35,7 -35,7 +35,7 @@@ use lightning::chain::chainmonitor
  use lightning::chain::transaction::OutPoint;
  use lightning::chain::keysinterface::{InMemorySigner, Recipient, KeyMaterial, KeysInterface};
  use lightning::ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 -use lightning::ln::channelmanager::{ChainParameters, ChannelManager};
 +use lightning::ln::channelmanager::{ChainParameters, ChannelManager, PaymentId};
  use lightning::ln::peer_handler::{MessageHandler,PeerManager,SocketDescriptor,IgnoringMessageHandler};
  use lightning::ln::msgs::DecodeError;
  use lightning::ln::script::ShutdownScript;
@@@ -163,6 -163,7 +163,6 @@@ impl<'a> std::hash::Hash for Peer<'a> 
  }
  
  type ChannelMan = ChannelManager<
 -      EnforcingSigner,
        Arc<chainmonitor::ChainMonitor<EnforcingSigner, Arc<dyn chain::Filter>, Arc<TestBroadcaster>, Arc<FuzzEstimator>, Arc<dyn Logger>, Arc<TestPersister>>>,
        Arc<TestBroadcaster>, Arc<KeyProvider>, Arc<FuzzEstimator>, Arc<dyn Logger>>;
  type PeerMan<'a> = PeerManager<Peer<'a>, Arc<ChannelMan>, Arc<P2PGossipSync<Arc<NetworkGraph<Arc<dyn Logger>>>, Arc<dyn chain::Access>, Arc<dyn Logger>>>, IgnoringMessageHandler, Arc<dyn Logger>, IgnoringMessageHandler>;
@@@ -481,7 -482,7 +481,7 @@@ pub fn do_test(data: &[u8], logger: &Ar
                                sha.input(&payment_hash.0[..]);
                                payment_hash.0 = Sha256::from_engine(sha).into_inner();
                                payments_sent += 1;
 -                              match channelmanager.send_payment(&route, payment_hash, &None) {
 +                              match channelmanager.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)) {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
                                let mut payment_secret = PaymentSecret([0; 32]);
                                payment_secret.0[0..8].copy_from_slice(&be64_to_array(payments_sent));
                                payments_sent += 1;
 -                              match channelmanager.send_payment(&route, payment_hash, &Some(payment_secret)) {
 +                              match channelmanager.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)) {
                                        Ok(_) => {},
                                        Err(_) => return,
                                }
@@@ -796,7 -797,7 +796,7 @@@ mod tests 
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000000 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000000 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 11 020203e8 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4e00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // 030012 - inbound read from peer id 0 of len 18
                // 0064 03000000000000000000000000000000 - message header indicating message length 100
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000001 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000001 0000000000003e80 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 11 020203e8 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4e00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // - now respond to the update_fulfill_htlc+commitment_signed messages the client sent to peer 0
                // 030012 - inbound read from peer id 0 of len 18
                // 030012 - inbound read from peer id 0 of len 18
                // 05ac 03000000000000000000000000000000 - message header indicating message length 1452
                // 0300ff - inbound read from peer id 0 of len 255
-               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000002 00000000000b0838 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 0000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
+               // 0080 3d00000000000000000000000000000000000000000000000000000000000000 0000000000000002 00000000000b0838 ff00000000000000000000000000000000000000000000000000000000000000 000003f0 00 030000000000000000000000000000000000000000000000000000000000000555 12 02030927c0 0401a0 060800000e0000010000 0a00000000000000000000000000000000000000000000000000000000000000 ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff - beginning of update_add_htlc from 0 to 1 via client
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300ff - inbound read from peer id 0 of len 255
                // 0300ff - inbound read from peer id 0 of len 255
                // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff
                // 0300c1 - inbound read from peer id 0 of len 193
-               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 4b00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
+               // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 5300000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac
                //
                // 030012 - inbound read from peer id 0 of len 18
                // 00a4 03000000000000000000000000000000 - message header indicating message length 164
                // - client now fails the HTLC backwards as it was unable to extract the payment preimage (CHECK 9 duplicate and CHECK 10)
  
                let logger = Arc::new(TrackingLogger { lines: Mutex::new(HashMap::new()) });
-               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4b000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
+               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005551202030927c00401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff53000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
  
                let log_entries = logger.lines.lock().unwrap();
                assert_eq!(log_entries.get(&("lightning::ln::peer_handler".to_string(), "Handling SendAcceptChannel event in peer_handler for node 030000000000000000000000000000000000000000000000000000000000000002 for channel ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679".to_string())), Some(&1)); // 1
index 06ef2aa0dc3422fc457ee05504c395e96f00eedc,28074ea4e057328bff7ba110f67ebd4215532e47..ae8625d89f4f60eaba2fe8127a5b957a75b5d301
@@@ -51,9 -51,9 +51,9 @@@ use crate::ln::msgs
  use crate::ln::onion_utils;
  use crate::ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, MAX_VALUE_MSAT};
  use crate::ln::wire::Encode;
 -use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, InMemorySigner, Recipient};
 +use crate::chain::keysinterface::{Sign, KeysInterface, KeysManager, Recipient};
  use crate::util::config::{UserConfig, ChannelConfig};
 -use crate::util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
 +use crate::util::events::{Event, EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason, HTLCDestination};
  use crate::util::{byte_utils, events};
  use crate::util::wakers::{Future, Notifier};
  use crate::util::scid_utils::fake_scid;
@@@ -66,7 -66,7 +66,7 @@@ use crate::prelude::*
  use core::{cmp, mem};
  use core::cell::RefCell;
  use crate::io::Read;
 -use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard};
 +use crate::sync::{Arc, Mutex, MutexGuard, RwLock, RwLockReadGuard, FairRwLock};
  use core::sync::atomic::{AtomicUsize, Ordering};
  use core::time::Duration;
  use core::ops::Deref;
@@@ -112,8 -112,7 +112,8 @@@ pub(super) struct PendingHTLCInfo 
        pub(super) routing: PendingHTLCRouting,
        pub(super) incoming_shared_secret: [u8; 32],
        payment_hash: PaymentHash,
 -      pub(super) amt_to_forward: u64,
 +      pub(super) incoming_amt_msat: Option<u64>, // Added in 0.0.113
 +      pub(super) outgoing_amt_msat: u64,
        pub(super) outgoing_cltv_value: u32,
  }
  
@@@ -130,22 -129,20 +130,22 @@@ pub(super) enum PendingHTLCStatus 
        Fail(HTLCFailureMsg),
  }
  
 -pub(super) enum HTLCForwardInfo {
 -      AddHTLC {
 -              forward_info: PendingHTLCInfo,
 +pub(super) struct PendingAddHTLCInfo {
 +      pub(super) forward_info: PendingHTLCInfo,
  
 -              // These fields are produced in `forward_htlcs()` and consumed in
 -              // `process_pending_htlc_forwards()` for constructing the
 -              // `HTLCSource::PreviousHopData` for failed and forwarded
 -              // HTLCs.
 -              //
 -              // Note that this may be an outbound SCID alias for the associated channel.
 -              prev_short_channel_id: u64,
 -              prev_htlc_id: u64,
 -              prev_funding_outpoint: OutPoint,
 -      },
 +      // These fields are produced in `forward_htlcs()` and consumed in
 +      // `process_pending_htlc_forwards()` for constructing the
 +      // `HTLCSource::PreviousHopData` for failed and forwarded
 +      // HTLCs.
 +      //
 +      // Note that this may be an outbound SCID alias for the associated channel.
 +      prev_short_channel_id: u64,
 +      prev_htlc_id: u64,
 +      prev_funding_outpoint: OutPoint,
 +}
 +
 +pub(super) enum HTLCForwardInfo {
 +      AddHTLC(PendingAddHTLCInfo),
        FailHTLC {
                htlc_id: u64,
                err_packet: msgs::OnionErrorPacket,
@@@ -398,6 -395,12 +398,6 @@@ pub(super) enum RAACommitmentOrder 
  // Note this is only exposed in cfg(test):
  pub(super) struct ChannelHolder<Signer: Sign> {
        pub(super) by_id: HashMap<[u8; 32], Channel<Signer>>,
 -      /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
 -      ///
 -      /// Outbound SCID aliases are added here once the channel is available for normal use, with
 -      /// SCIDs being added once the funding transaction is confirmed at the channel's required
 -      /// confirmation depth.
 -      pub(super) short_to_chan_info: HashMap<u64, (PublicKey, [u8; 32])>,
        /// Map from payment hash to the payment data and any HTLCs which are to us and can be
        /// failed/claimed by the user.
        ///
@@@ -470,7 -473,6 +470,7 @@@ pub(crate) enum PendingOutboundPayment 
        Fulfilled {
                session_privs: HashSet<[u8; 32]>,
                payment_hash: Option<PaymentHash>,
 +              timer_ticks_without_htlcs: u8,
        },
        /// When a payer gives up trying to retry a payment, they inform us, letting us generate a
        /// `PaymentFailed` event when all HTLCs have irrevocably failed. This avoids a number of race
  }
  
  impl PendingOutboundPayment {
 -      fn is_retryable(&self) -> bool {
 -              match self {
 -                      PendingOutboundPayment::Retryable { .. } => true,
 -                      _ => false,
 -              }
 -      }
        fn is_fulfilled(&self) -> bool {
                match self {
                        PendingOutboundPayment::Fulfilled { .. } => true,
                                => session_privs,
                });
                let payment_hash = self.payment_hash();
 -              *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash };
 +              *self = PendingOutboundPayment::Fulfilled { session_privs, payment_hash, timer_ticks_without_htlcs: 0 };
        }
  
        fn mark_abandoned(&mut self) -> Result<(), ()> {
  /// concrete type of the KeysManager.
  ///
  /// (C-not exported) as Arcs don't make sense in bindings
 -pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<InMemorySigner, Arc<M>, Arc<T>, Arc<KeysManager>, Arc<F>, Arc<L>>;
 +pub type SimpleArcChannelManager<M, T, F, L> = ChannelManager<Arc<M>, Arc<T>, Arc<KeysManager>, Arc<F>, Arc<L>>;
  
  /// SimpleRefChannelManager is a type alias for a ChannelManager reference, and is the reference
  /// counterpart to the SimpleArcChannelManager type alias. Use this type by default when you don't
  /// concrete type of the KeysManager.
  ///
  /// (C-not exported) as Arcs don't make sense in bindings
 -pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManager<InMemorySigner, &'a M, &'b T, &'c KeysManager, &'d F, &'e L>;
 +pub type SimpleRefChannelManager<'a, 'b, 'c, 'd, 'e, M, T, F, L> = ChannelManager<&'a M, &'b T, &'c KeysManager, &'d F, &'e L>;
  
  /// Manager which keeps track of a number of channels and sends messages to the appropriate
  /// channel, also tracking HTLC preimages and forwarding onion packets appropriately.
  //  |   |
  //  |   |__`id_to_peer`
  //  |   |
 +//  |   |__`short_to_chan_info`
 +//  |   |
  //  |   |__`per_peer_state`
  //  |       |
  //  |       |__`outbound_scid_aliases`
  //  |                   |
  //  |                   |__`pending_background_events`
  //
 -pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      where M::Target: chain::Watch<Signer>,
 +pub struct ChannelManager<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
                                L::Target: Logger,
  {
  
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        #[cfg(any(test, feature = "_test_utils"))]
 -      pub(super) channel_state: Mutex<ChannelHolder<Signer>>,
 +      pub(super) channel_state: Mutex<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
        #[cfg(not(any(test, feature = "_test_utils")))]
 -      channel_state: Mutex<ChannelHolder<Signer>>,
 +      channel_state: Mutex<ChannelHolder<<K::Target as KeysInterface>::Signer>>,
  
        /// Storage for PaymentSecrets and any requirements on future inbound payments before we will
        /// expose them to users via a PaymentReceived event. HTLCs which do not meet the requirements
        /// See `ChannelManager` struct-level documentation for lock order requirements.
        id_to_peer: Mutex<HashMap<[u8; 32], PublicKey>>,
  
 +      /// SCIDs (and outbound SCID aliases) -> `counterparty_node_id`s and `channel_id`s.
 +      ///
 +      /// Outbound SCID aliases are added here once the channel is available for normal use, with
 +      /// SCIDs being added once the funding transaction is confirmed at the channel's required
 +      /// confirmation depth.
 +      ///
 +      /// Note that while this holds `counterparty_node_id`s and `channel_id`s, no consistency
 +      /// guarantees are made about the existence of a peer with the `counterparty_node_id` nor a
 +      /// channel with the `channel_id` in our other maps.
 +      ///
 +      /// See `ChannelManager` struct-level documentation for lock order requirements.
 +      #[cfg(test)]
 +      pub(super) short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
 +      #[cfg(not(test))]
 +      short_to_chan_info: FairRwLock<HashMap<u64, (PublicKey, [u8; 32])>>,
 +
        our_network_key: SecretKey,
        our_network_pubkey: PublicKey,
  
@@@ -969,14 -959,13 +969,14 @@@ const CHECK_CLTV_EXPIRY_SANITY: u32 = M
  #[allow(dead_code)]
  const CHECK_CLTV_EXPIRY_SANITY_2: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_GRACE_PERIOD_BLOCKS - 2*CLTV_CLAIM_BUFFER;
  
 -/// The number of blocks before we consider an outbound payment for expiry if it doesn't have any
 -/// pending HTLCs in flight.
 -pub(crate) const PAYMENT_EXPIRY_BLOCKS: u32 = 3;
 -
  /// The number of ticks of [`ChannelManager::timer_tick_occurred`] until expiry of incomplete MPPs
  pub(crate) const MPP_TIMEOUT_TICKS: u8 = 3;
  
 +/// The number of ticks of [`ChannelManager::timer_tick_occurred`] until we time-out the
 +/// idempotency of payments by [`PaymentId`]. See
 +/// [`ChannelManager::remove_stale_resolved_payments`].
 +pub(crate) const IDEMPOTENCY_TIMEOUT_TICKS: u8 = 7;
 +
  /// Information needed for constructing an invoice route hint for this channel.
  #[derive(Clone, Debug, PartialEq)]
  pub struct CounterpartyForwardingInfo {
@@@ -1217,9 -1206,6 +1217,9 @@@ pub enum PaymentSendFailure 
        /// All paths which were attempted failed to send, with no channel state change taking place.
        /// You can freely retry the payment in full (though you probably want to do so over different
        /// paths than the ones selected).
 +      ///
 +      /// [`ChannelManager::abandon_payment`] does *not* need to be called for this payment and
 +      /// [`ChannelManager::retry_payment`] will *not* work for this payment.
        AllFailedRetrySafe(Vec<APIError>),
        /// Some paths which were attempted failed to send, though possibly not all. At least some
        /// paths have irrevocably committed to the HTLC and retrying the payment in full would result
@@@ -1310,11 -1296,9 +1310,11 @@@ macro_rules! handle_error 
  }
  
  macro_rules! update_maps_on_chan_removal {
 -      ($self: expr, $short_to_chan_info: expr, $channel: expr) => {
 +      ($self: expr, $channel: expr) => {{
 +              $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
 +              let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
                if let Some(short_id) = $channel.get_short_channel_id() {
 -                      $short_to_chan_info.remove(&short_id);
 +                      short_to_chan_info.remove(&short_id);
                } else {
                        // If the channel was never confirmed on-chain prior to its closure, remove the
                        // outbound SCID alias we used for it from the collision-prevention set. While we
                        let alias_removed = $self.outbound_scid_aliases.lock().unwrap().remove(&$channel.outbound_scid_alias());
                        debug_assert!(alias_removed);
                }
 -              $self.id_to_peer.lock().unwrap().remove(&$channel.channel_id());
 -              $short_to_chan_info.remove(&$channel.outbound_scid_alias());
 -      }
 +              short_to_chan_info.remove(&$channel.outbound_scid_alias());
 +      }}
  }
  
  /// Returns (boolean indicating if we should remove the Channel object from memory, a mapped error)
  macro_rules! convert_chan_err {
 -      ($self: ident, $err: expr, $short_to_chan_info: expr, $channel: expr, $channel_id: expr) => {
 +      ($self: ident, $err: expr, $channel: expr, $channel_id: expr) => {
                match $err {
                        ChannelError::Warn(msg) => {
                                (false, MsgHandleErrInternal::from_chan_no_close(ChannelError::Warn(msg), $channel_id.clone()))
                        },
                        ChannelError::Close(msg) => {
                                log_error!($self.logger, "Closing channel {} due to close-required error: {}", log_bytes!($channel_id[..]), msg);
 -                              update_maps_on_chan_removal!($self, $short_to_chan_info, $channel);
 +                              update_maps_on_chan_removal!($self, $channel);
                                let shutdown_res = $channel.force_shutdown(true);
                                (true, MsgHandleErrInternal::from_finish_shutdown(msg, *$channel_id, $channel.get_user_id(),
                                        shutdown_res, $self.get_channel_update_for_broadcast(&$channel).ok()))
  }
  
  macro_rules! break_chan_entry {
 -      ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
 +      ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
 -                              let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
 +                              let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
  }
  
  macro_rules! try_chan_entry {
 -      ($self: ident, $res: expr, $channel_state: expr, $entry: expr) => {
 +      ($self: ident, $res: expr, $entry: expr) => {
                match $res {
                        Ok(res) => res,
                        Err(e) => {
 -                              let (drop, res) = convert_chan_err!($self, e, $channel_state.short_to_chan_info, $entry.get_mut(), $entry.key());
 +                              let (drop, res) = convert_chan_err!($self, e, $entry.get_mut(), $entry.key());
                                if drop {
                                        $entry.remove_entry();
                                }
  }
  
  macro_rules! remove_channel {
 -      ($self: expr, $channel_state: expr, $entry: expr) => {
 +      ($self: expr, $entry: expr) => {
                {
                        let channel = $entry.remove_entry().1;
 -                      update_maps_on_chan_removal!($self, $channel_state.short_to_chan_info, channel);
 +                      update_maps_on_chan_removal!($self, channel);
                        channel
                }
        }
  }
  
  macro_rules! handle_monitor_update_res {
 -      ($self: ident, $err: expr, $short_to_chan_info: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
 +      ($self: ident, $err: expr, $chan: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr, $chan_id: expr) => {
                match $err {
                        ChannelMonitorUpdateStatus::PermanentFailure => {
                                log_error!($self.logger, "Closing channel {} due to monitor update ChannelMonitorUpdateStatus::PermanentFailure", log_bytes!($chan_id[..]));
 -                              update_maps_on_chan_removal!($self, $short_to_chan_info, $chan);
 +                              update_maps_on_chan_removal!($self, $chan);
                                // TODO: $failed_fails is dropped here, which will cause other channels to hit the
                                // chain in a confused state! We need to move them into the ChannelMonitor which
                                // will be responsible for failing backwards once things confirm on-chain.
                        },
                }
        };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
 -              let (res, drop) = handle_monitor_update_res!($self, $err, $channel_state.short_to_chan_info, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $resend_channel_ready: expr, $failed_forwards: expr, $failed_fails: expr, $failed_finalized_fulfills: expr) => { {
 +              let (res, drop) = handle_monitor_update_res!($self, $err, $entry.get_mut(), $action_type, $resend_raa, $resend_commitment, $resend_channel_ready, $failed_forwards, $failed_fails, $failed_finalized_fulfills, $entry.key());
                if drop {
                        $entry.remove_entry();
                }
                res
        } };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, COMMITMENT_UPDATE_ONLY) => { {
                debug_assert!($action_type == RAACommitmentOrder::CommitmentFirst);
 -              handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
 +              handle_monitor_update_res!($self, $err, $entry, $action_type, false, true, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        } };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
 -              handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $chan_id: expr, NO_UPDATE) => {
 +              handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, false, Vec::new(), Vec::new(), Vec::new(), $chan_id)
        };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
 -              handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_channel_ready: expr, OPTIONALLY_RESEND_FUNDING_LOCKED) => {
 +              handle_monitor_update_res!($self, $err, $entry, $action_type, false, false, $resend_channel_ready, Vec::new(), Vec::new(), Vec::new())
        };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
 -              handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr) => {
 +              handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, Vec::new(), Vec::new(), Vec::new())
        };
 -      ($self: ident, $err: expr, $channel_state: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
 -              handle_monitor_update_res!($self, $err, $channel_state, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
 +      ($self: ident, $err: expr, $entry: expr, $action_type: path, $resend_raa: expr, $resend_commitment: expr, $failed_forwards: expr, $failed_fails: expr) => {
 +              handle_monitor_update_res!($self, $err, $entry, $action_type, $resend_raa, $resend_commitment, false, $failed_forwards, $failed_fails, Vec::new())
        };
  }
  
  macro_rules! send_channel_ready {
 -      ($short_to_chan_info: expr, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {
 +      ($self: ident, $pending_msg_events: expr, $channel: expr, $channel_ready_msg: expr) => {{
                $pending_msg_events.push(events::MessageSendEvent::SendChannelReady {
                        node_id: $channel.get_counterparty_node_id(),
                        msg: $channel_ready_msg,
                });
                // Note that we may send a `channel_ready` multiple times for a channel if we reconnect, so
                // we allow collisions, but we shouldn't ever be updating the channel ID pointed to.
 -              let outbound_alias_insert = $short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
 +              let mut short_to_chan_info = $self.short_to_chan_info.write().unwrap();
 +              let outbound_alias_insert = short_to_chan_info.insert($channel.outbound_scid_alias(), ($channel.get_counterparty_node_id(), $channel.channel_id()));
                assert!(outbound_alias_insert.is_none() || outbound_alias_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                        "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                if let Some(real_scid) = $channel.get_short_channel_id() {
 -                      let scid_insert = $short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
 +                      let scid_insert = short_to_chan_info.insert(real_scid, ($channel.get_counterparty_node_id(), $channel.channel_id()));
                        assert!(scid_insert.is_none() || scid_insert.unwrap() == ($channel.get_counterparty_node_id(), $channel.channel_id()),
                                "SCIDs should never collide - ensure you weren't behind the chain tip by a full month when creating channels");
                }
 +      }}
 +}
 +
 +macro_rules! emit_channel_ready_event {
 +      ($self: expr, $channel: expr) => {
 +              if $channel.should_emit_channel_ready_event() {
 +                      {
 +                              let mut pending_events = $self.pending_events.lock().unwrap();
 +                              pending_events.push(events::Event::ChannelReady {
 +                                      channel_id: $channel.channel_id(),
 +                                      user_channel_id: $channel.get_user_id(),
 +                                      counterparty_node_id: $channel.get_counterparty_node_id(),
 +                                      channel_type: $channel.get_channel_type().clone(),
 +                              });
 +                      }
 +                      $channel.set_channel_ready_event_emitted();
 +              }
        }
  }
  
@@@ -1533,7 -1500,7 +1533,7 @@@ macro_rules! handle_chan_restoration_lo
                                // Similar to the above, this implies that we're letting the channel_ready fly
                                // before it should be allowed to.
                                assert!(chanmon_update.is_none());
 -                              send_channel_ready!($channel_state.short_to_chan_info, $channel_state.pending_msg_events, $channel_entry.get(), msg);
 +                              send_channel_ready!($self, $channel_state.pending_msg_events, $channel_entry.get(), msg);
                        }
                        if let Some(msg) = $announcement_sigs {
                                $channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
                                });
                        }
  
 +                      emit_channel_ready_event!($self, $channel_entry.get_mut());
 +
                        let funding_broadcastable: Option<Transaction> = $funding_broadcastable; // Force type-checking to resolve
                        if let Some(monitor_update) = chanmon_update {
                                // We only ever broadcast a funding transaction in response to a funding_signed
                                                if $raa.is_none() {
                                                        order = RAACommitmentOrder::CommitmentFirst;
                                                }
 -                                              break handle_monitor_update_res!($self, e, $channel_state, $channel_entry, order, $raa.is_some(), true);
 +                                              break handle_monitor_update_res!($self, e, $channel_entry, order, $raa.is_some(), true);
                                        }
                                }
                        }
@@@ -1627,10 -1592,10 +1627,10 @@@ macro_rules! post_handle_chan_restorati
        } }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<Signer, M, T, K, F, L>
 -      where M::Target: chain::Watch<Signer>,
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F, L>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
  
                        channel_state: Mutex::new(ChannelHolder{
                                by_id: HashMap::new(),
 -                              short_to_chan_info: HashMap::new(),
                                claimable_htlcs: HashMap::new(),
                                pending_msg_events: Vec::new(),
                        }),
                        pending_outbound_payments: Mutex::new(HashMap::new()),
                        forward_htlcs: Mutex::new(HashMap::new()),
                        id_to_peer: Mutex::new(HashMap::new()),
 +                      short_to_chan_info: FairRwLock::new(HashMap::new()),
  
                        our_network_key: keys_manager.get_node_secret(Recipient::Node).unwrap(),
                        our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &keys_manager.get_node_secret(Recipient::Node).unwrap()),
                Ok(temporary_channel_id)
        }
  
 -      fn list_channels_with_filter<Fn: FnMut(&(&[u8; 32], &Channel<Signer>)) -> bool>(&self, f: Fn) -> Vec<ChannelDetails> {
 +      fn list_channels_with_filter<Fn: FnMut(&(&[u8; 32], &Channel<<K::Target as KeysInterface>::Signer>)) -> bool>(&self, f: Fn) -> Vec<ChannelDetails> {
                let mut res = Vec::new();
                {
                        let channel_state = self.channel_state.lock().unwrap();
        }
  
        /// Helper function that issues the channel close events
 -      fn issue_channel_close_events(&self, channel: &Channel<Signer>, closure_reason: ClosureReason) {
 +      fn issue_channel_close_events(&self, channel: &Channel<<K::Target as KeysInterface>::Signer>, closure_reason: ClosureReason) {
                let mut pending_events_lock = self.pending_events.lock().unwrap();
                match channel.unbroadcasted_funding() {
                        Some(transaction) => {
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
 -                                                      handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
 +                                                      handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
 -                                                      remove_channel!(self, channel_state, chan_entry);
 +                                                      remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
                                        });
  
                                        if chan_entry.get().is_shutdown() {
 -                                              let channel = remove_channel!(self, channel_state, chan_entry);
 +                                              let channel = remove_channel!(self, chan_entry);
                                                if let Ok(channel_update) = self.get_channel_update_for_broadcast(&channel) {
                                                        channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
                                                                msg: channel_update
                                } else {
                                        self.issue_channel_close_events(chan.get(),ClosureReason::HolderForceClosed);
                                }
 -                              remove_channel!(self, channel_state, chan)
 +                              remove_channel!(self, chan)
                        } else {
                                return Err(APIError::ChannelUnavailable{err: "No such channel".to_owned()});
                        }
                }
  
                let routing = match hop_data.format {
-                       msgs::OnionHopDataFormat::Legacy { .. } => {
-                               return Err(ReceiveError {
-                                       err_code: 0x4000|0x2000|3,
-                                       err_data: Vec::new(),
-                                       msg: "We require payment_secrets",
-                               });
-                       },
                        msgs::OnionHopDataFormat::NonFinalNode { .. } => {
                                return Err(ReceiveError {
                                        err_code: 0x4000|22,
                        routing,
                        payment_hash,
                        incoming_shared_secret: shared_secret,
 -                      amt_to_forward: amt_msat,
 +                      incoming_amt_msat: Some(amt_msat),
 +                      outgoing_amt_msat: amt_msat,
                        outgoing_cltv_value: hop_data.outgoing_cltv_value,
                })
        }
                                };
  
                                let short_channel_id = match next_hop_data.format {
-                                       msgs::OnionHopDataFormat::Legacy { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::NonFinalNode { short_channel_id } => short_channel_id,
                                        msgs::OnionHopDataFormat::FinalNode { .. } => {
                                                return_err!("Final Node OnionHopData provided for us as an intermediary node", 0x4000 | 22, &[0;0]);
                                        },
                                        payment_hash: msg.payment_hash.clone(),
                                        incoming_shared_secret: shared_secret,
 -                                      amt_to_forward: next_hop_data.amt_to_forward,
 +                                      incoming_amt_msat: Some(msg.amount_msat),
 +                                      outgoing_amt_msat: next_hop_data.amt_to_forward,
                                        outgoing_cltv_value: next_hop_data.outgoing_cltv_value,
                                })
                        }
                };
  
 -              if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref amt_to_forward, ref outgoing_cltv_value, .. }) = &pending_forward_info {
 +              if let &PendingHTLCStatus::Forward(PendingHTLCInfo { ref routing, ref outgoing_amt_msat, ref outgoing_cltv_value, .. }) = &pending_forward_info {
                        // If short_channel_id is 0 here, we'll reject the HTLC as there cannot be a channel
                        // with a short_channel_id of 0. This is important as various things later assume
                        // short_channel_id is non-0 in any ::Forward.
                        if let &PendingHTLCRouting::Forward { ref short_channel_id, .. } = routing {
                                if let Some((err, code, chan_update)) = loop {
 +                                      let id_option = self.short_to_chan_info.read().unwrap().get(&short_channel_id).cloned();
                                        let mut channel_state = self.channel_state.lock().unwrap();
 -                                      let id_option = channel_state.short_to_chan_info.get(&short_channel_id).cloned();
                                        let forwarding_id_opt = match id_option {
                                                None => { // unknown_next_peer
                                                        // Note that this is likely a timing oracle for detecting whether an scid is a
                                                        // phantom.
 -                                                      if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id) {
 +                                                      if fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, *short_channel_id, &self.genesis_hash) {
                                                                None
                                                        } else {
                                                                break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
                                                Some((_cp_id, chan_id)) => Some(chan_id.clone()),
                                        };
                                        let chan_update_opt = if let Some(forwarding_id) = forwarding_id_opt {
 -                                              let chan = channel_state.by_id.get_mut(&forwarding_id).unwrap();
 +                                              let chan = match channel_state.by_id.get_mut(&forwarding_id) {
 +                                                      None => {
 +                                                              // Channel was removed. The short_to_chan_info and by_id maps have
 +                                                              // no consistency guarantees.
 +                                                              break Some(("Don't have available channel for forwarding as requested.", 0x4000 | 10, None));
 +                                                      },
 +                                                      Some(chan) => chan
 +                                              };
                                                if !chan.should_announce() && !self.default_configuration.accept_forwards_to_priv_channels {
                                                        // Note that the behavior here should be identical to the above block - we
                                                        // should NOT reveal the existence or non-existence of a private channel if
                                                if !chan.is_live() { // channel_disabled
                                                        break Some(("Forwarding channel is not in a ready state.", 0x1000 | 20, chan_update_opt));
                                                }
 -                                              if *amt_to_forward < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
 +                                              if *outgoing_amt_msat < chan.get_counterparty_htlc_minimum_msat() { // amount_below_minimum
                                                        break Some(("HTLC amount was below the htlc_minimum_msat", 0x1000 | 11, chan_update_opt));
                                                }
 -                                              if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *amt_to_forward, *outgoing_cltv_value) {
 +                                              if let Err((err, code)) = chan.htlc_satisfies_config(&msg, *outgoing_amt_msat, *outgoing_cltv_value) {
                                                        break Some((err, code, chan_update_opt));
                                                }
                                                chan_update_opt
        /// [`MessageSendEvent::BroadcastChannelUpdate`] event.
        ///
        /// May be called with channel_state already locked!
 -      fn get_channel_update_for_broadcast(&self, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
 +      fn get_channel_update_for_broadcast(&self, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                if !chan.should_announce() {
                        return Err(LightningError {
                                err: "Cannot broadcast a channel_update for a private channel".to_owned(),
        /// and thus MUST NOT be called unless the recipient of the resulting message has already
        /// provided evidence that they know about the existence of the channel.
        /// May be called with channel_state already locked!
 -      fn get_channel_update_for_unicast(&self, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
 +      fn get_channel_update_for_unicast(&self, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                log_trace!(self.logger, "Attempting to generate channel update for channel {}", log_bytes!(chan.channel_id()));
                let short_channel_id = match chan.get_short_channel_id().or(chan.latest_inbound_scid_alias()) {
                        None => return Err(LightningError{err: "Channel not yet established".to_owned(), action: msgs::ErrorAction::IgnoreError}),
  
                self.get_channel_update_for_onion(short_channel_id, chan)
        }
 -      fn get_channel_update_for_onion(&self, short_channel_id: u64, chan: &Channel<Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
 +      fn get_channel_update_for_onion(&self, short_channel_id: u64, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> Result<msgs::ChannelUpdate, LightningError> {
                log_trace!(self.logger, "Generating channel update for channel {}", log_bytes!(chan.channel_id()));
                let were_node_one = PublicKey::from_secret_key(&self.secp_ctx, &self.our_network_key).serialize()[..] < chan.get_counterparty_node_id().serialize()[..];
  
        }
  
        // Only public for testing, this should otherwise never be called direcly
 -      pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_params: &Option<PaymentParameters>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>) -> Result<(), APIError> {
 +      pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_params: &Option<PaymentParameters>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, payment_id: PaymentId, keysend_preimage: &Option<PaymentPreimage>, session_priv_bytes: [u8; 32]) -> Result<(), APIError> {
                log_trace!(self.logger, "Attempting to send payment for path with next hop {}", path.first().unwrap().short_channel_id);
                let prng_seed = self.keys_manager.get_secure_random_bytes();
 -              let session_priv_bytes = self.keys_manager.get_secure_random_bytes();
                let session_priv = SecretKey::from_slice(&session_priv_bytes[..]).expect("RNG is busted");
  
                let onion_keys = onion_utils::construct_onion_keys(&self.secp_ctx, &path, &session_priv)
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
  
                let err: Result<(), _> = loop {
 -                      let mut channel_lock = self.channel_state.lock().unwrap();
 -
 -                      let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
 -                      let payment_entry = pending_outbounds.entry(payment_id);
 -                      if let hash_map::Entry::Occupied(payment) = &payment_entry {
 -                              if !payment.get().is_retryable() {
 -                                      return Err(APIError::RouteError {
 -                                              err: "Payment already completed"
 -                                      });
 -                              }
 -                      }
 -
 -                      let id = match channel_lock.short_to_chan_info.get(&path.first().unwrap().short_channel_id) {
 +                      let id = match self.short_to_chan_info.read().unwrap().get(&path.first().unwrap().short_channel_id) {
                                None => return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()}),
                                Some((_cp_id, chan_id)) => chan_id.clone(),
                        };
  
 -                      macro_rules! insert_outbound_payment {
 -                              () => {
 -                                      let payment = payment_entry.or_insert_with(|| PendingOutboundPayment::Retryable {
 -                                              session_privs: HashSet::new(),
 -                                              pending_amt_msat: 0,
 -                                              pending_fee_msat: Some(0),
 -                                              payment_hash: *payment_hash,
 -                                              payment_secret: *payment_secret,
 -                                              starting_block_height: self.best_block.read().unwrap().height(),
 -                                              total_msat: total_value,
 -                                      });
 -                                      assert!(payment.insert(session_priv_bytes, path));
 -                              }
 -                      }
 -
 +                      let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
                        if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(id) {
                                match {
                                                        payment_secret: payment_secret.clone(),
                                                        payment_params: payment_params.clone(),
                                                }, onion_packet, &self.logger),
 -                                      channel_state, chan)
 +                                              chan)
                                } {
                                        Some((update_add, commitment_signed, monitor_update)) => {
                                                let update_err = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
                                                let chan_id = chan.get().channel_id();
                                                match (update_err,
 -                                                      handle_monitor_update_res!(self, update_err, channel_state, chan,
 +                                                      handle_monitor_update_res!(self, update_err, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false, true))
                                                {
                                                        (ChannelMonitorUpdateStatus::PermanentFailure, Err(e)) => break Err(e),
 -                                                      (ChannelMonitorUpdateStatus::Completed, Ok(())) => {
 -                                                              insert_outbound_payment!();
 -                                                      },
 +                                                      (ChannelMonitorUpdateStatus::Completed, Ok(())) => {},
                                                        (ChannelMonitorUpdateStatus::InProgress, Err(_)) => {
                                                                // Note that MonitorUpdateInProgress here indicates (per function
                                                                // docs) that we will resend the commitment update once monitor
                                                                // indicating that it is unsafe to retry the payment wholesale,
                                                                // which we do in the send_payment check for
                                                                // MonitorUpdateInProgress, below.
 -                                                              insert_outbound_payment!(); // Only do this after possibly break'ing on Perm failure above.
                                                                return Err(APIError::MonitorUpdateInProgress);
                                                        },
                                                        _ => unreachable!(),
                                                        },
                                                });
                                        },
 -                                      None => { insert_outbound_payment!(); },
 +                                      None => { },
                                }
 -                      } else { unreachable!(); }
 +                      } else {
 +                              // The channel was likely removed after we fetched the id from the
 +                              // `short_to_chan_info` map, but before we successfully locked the `by_id` map.
 +                              // This can occur as no consistency guarantees exists between the two maps.
 +                              return Err(APIError::ChannelUnavailable{err: "No channel available with first hop!".to_owned()});
 +                      }
                        return Ok(());
                };
  
        /// Value parameters are provided via the last hop in route, see documentation for RouteHop
        /// fields for more info.
        ///
 -      /// Note that if the payment_hash already exists elsewhere (eg you're sending a duplicative
 -      /// payment), we don't do anything to stop you! We always try to ensure that if the provided
 -      /// next hop knows the preimage to payment_hash they can claim an additional amount as
 -      /// specified in the last hop in the route! Thus, you should probably do your own
 -      /// payment_preimage tracking (which you should already be doing as they represent "proof of
 -      /// payment") and prevent double-sends yourself.
 +      /// If a pending payment is currently in-flight with the same [`PaymentId`] provided, this
 +      /// method will error with an [`APIError::RouteError`]. Note, however, that once a payment
 +      /// is no longer pending (either via [`ChannelManager::abandon_payment`], or handling of an
 +      /// [`Event::PaymentSent`]) LDK will not stop you from sending a second payment with the same
 +      /// [`PaymentId`].
 +      ///
 +      /// Thus, in order to ensure duplicate payments are not sent, you should implement your own
 +      /// tracking of payments, including state to indicate once a payment has completed. Because you
 +      /// should also ensure that [`PaymentHash`]es are not re-used, for simplicity, you should
 +      /// consider using the [`PaymentHash`] as the key for tracking payments. In that case, the
 +      /// [`PaymentId`] should be a copy of the [`PaymentHash`] bytes.
        ///
 -      /// May generate SendHTLCs message(s) event on success, which should be relayed.
 +      /// May generate SendHTLCs message(s) event on success, which should be relayed (e.g. via
 +      /// [`PeerManager::process_events`]).
        ///
        /// Each path may have a different return value, and PaymentSendValue may return a Vec with
        /// each entry matching the corresponding-index entry in the route paths, see
        /// newer nodes, it will be provided to you in the invoice. If you do not have one, the Route
        /// must not contain multiple paths as multi-path payments require a recipient-provided
        /// payment_secret.
 +      ///
        /// If a payment_secret *is* provided, we assume that the invoice had the payment_secret feature
        /// bit set (either as required or as available). If multiple paths are present in the Route,
        /// we assume the invoice had the basic_mpp feature set.
 -      pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>) -> Result<PaymentId, PaymentSendFailure> {
 -              self.send_payment_internal(route, payment_hash, payment_secret, None, None, None)
 +      ///
 +      /// [`Event::PaymentSent`]: events::Event::PaymentSent
 +      /// [`PeerManager::process_events`]: crate::ln::peer_handler::PeerManager::process_events
 +      pub fn send_payment(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, payment_id: PaymentId) -> Result<(), PaymentSendFailure> {
 +              let onion_session_privs = self.add_new_pending_payment(payment_hash, *payment_secret, payment_id, route)?;
 +              self.send_payment_internal(route, payment_hash, payment_secret, None, payment_id, None, onion_session_privs)
 +      }
 +
 +      #[cfg(test)]
 +      pub(crate) fn test_add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
 +              self.add_new_pending_payment(payment_hash, payment_secret, payment_id, route)
        }
  
 -      fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: Option<PaymentId>, recv_value_msat: Option<u64>) -> Result<PaymentId, PaymentSendFailure> {
 +      fn add_new_pending_payment(&self, payment_hash: PaymentHash, payment_secret: Option<PaymentSecret>, payment_id: PaymentId, route: &Route) -> Result<Vec<[u8; 32]>, PaymentSendFailure> {
 +              let mut onion_session_privs = Vec::with_capacity(route.paths.len());
 +              for _ in 0..route.paths.len() {
 +                      onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
 +              }
 +
 +              let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
 +              match pending_outbounds.entry(payment_id) {
 +                      hash_map::Entry::Occupied(_) => Err(PaymentSendFailure::ParameterError(APIError::RouteError {
 +                              err: "Payment already in progress"
 +                      })),
 +                      hash_map::Entry::Vacant(entry) => {
 +                              let payment = entry.insert(PendingOutboundPayment::Retryable {
 +                                      session_privs: HashSet::new(),
 +                                      pending_amt_msat: 0,
 +                                      pending_fee_msat: Some(0),
 +                                      payment_hash,
 +                                      payment_secret,
 +                                      starting_block_height: self.best_block.read().unwrap().height(),
 +                                      total_msat: route.get_total_amount(),
 +                              });
 +
 +                              for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
 +                                      assert!(payment.insert(*session_priv_bytes, path));
 +                              }
 +
 +                              Ok(onion_session_privs)
 +                      },
 +              }
 +      }
 +
 +      fn send_payment_internal(&self, route: &Route, payment_hash: PaymentHash, payment_secret: &Option<PaymentSecret>, keysend_preimage: Option<PaymentPreimage>, payment_id: PaymentId, recv_value_msat: Option<u64>, onion_session_privs: Vec<[u8; 32]>) -> Result<(), PaymentSendFailure> {
                if route.paths.len() < 1 {
                        return Err(PaymentSendFailure::ParameterError(APIError::RouteError{err: "There must be at least one path to send over"}));
                }
                let mut total_value = 0;
                let our_node_id = self.get_our_node_id();
                let mut path_errs = Vec::with_capacity(route.paths.len());
 -              let payment_id = if let Some(id) = payment_id { id } else { PaymentId(self.keys_manager.get_secure_random_bytes()) };
                'path_check: for path in route.paths.iter() {
                        if path.len() < 1 || path.len() > 20 {
                                path_errs.push(Err(APIError::RouteError{err: "Path didn't go anywhere/had bogus size"}));
  
                let cur_height = self.best_block.read().unwrap().height() + 1;
                let mut results = Vec::new();
 -              for path in route.paths.iter() {
 -                      results.push(self.send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage));
 +              debug_assert_eq!(route.paths.len(), onion_session_privs.len());
 +              for (path, session_priv) in route.paths.iter().zip(onion_session_privs.into_iter()) {
 +                      let mut path_res = self.send_payment_along_path(&path, &route.payment_params, &payment_hash, payment_secret, total_value, cur_height, payment_id, &keysend_preimage, session_priv);
 +                      match path_res {
 +                              Ok(_) => {},
 +                              Err(APIError::MonitorUpdateInProgress) => {
 +                                      // While a MonitorUpdateInProgress is an Err(_), the payment is still
 +                                      // considered "in flight" and we shouldn't remove it from the
 +                                      // PendingOutboundPayment set.
 +                              },
 +                              Err(_) => {
 +                                      let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
 +                                      if let Some(payment) = pending_outbounds.get_mut(&payment_id) {
 +                                              let removed = payment.remove(&session_priv, Some(path));
 +                                              debug_assert!(removed, "This can't happen as the payment has an entry for this path added by callers");
 +                                      } else {
 +                                              debug_assert!(false, "This can't happen as the payment was added by callers");
 +                                              path_res = Err(APIError::APIMisuseError { err: "Internal error: payment disappeared during processing. Please report this bug!".to_owned() });
 +                                      }
 +                              }
 +                      }
 +                      results.push(path_res);
                }
                let mut has_ok = false;
                let mut has_err = false;
                                } else { None },
                        })
                } else if has_err {
 -                      // If we failed to send any paths, we shouldn't have inserted the new PaymentId into
 -                      // our `pending_outbound_payments` map at all.
 -                      debug_assert!(self.pending_outbound_payments.lock().unwrap().get(&payment_id).is_none());
 +                      // If we failed to send any paths, we should remove the new PaymentId from the
 +                      // `pending_outbound_payments` map, as the user isn't expected to `abandon_payment`.
 +                      let removed = self.pending_outbound_payments.lock().unwrap().remove(&payment_id).is_some();
 +                      debug_assert!(removed, "We should always have a pending payment to remove here");
                        Err(PaymentSendFailure::AllFailedRetrySafe(results.drain(..).map(|r| r.unwrap_err()).collect()))
                } else {
 -                      Ok(payment_id)
 +                      Ok(())
                }
        }
  
                        }
                }
  
 +              let mut onion_session_privs = Vec::with_capacity(route.paths.len());
 +              for _ in 0..route.paths.len() {
 +                      onion_session_privs.push(self.keys_manager.get_secure_random_bytes());
 +              }
 +
                let (total_msat, payment_hash, payment_secret) = {
 -                      let outbounds = self.pending_outbound_payments.lock().unwrap();
 -                      if let Some(payment) = outbounds.get(&payment_id) {
 -                              match payment {
 -                                      PendingOutboundPayment::Retryable {
 -                                              total_msat, payment_hash, payment_secret, pending_amt_msat, ..
 -                                      } => {
 -                                              let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
 -                                              if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
 +                      let mut outbounds = self.pending_outbound_payments.lock().unwrap();
 +                      match outbounds.get_mut(&payment_id) {
 +                              Some(payment) => {
 +                                      let res = match payment {
 +                                              PendingOutboundPayment::Retryable {
 +                                                      total_msat, payment_hash, payment_secret, pending_amt_msat, ..
 +                                              } => {
 +                                                      let retry_amt_msat: u64 = route.paths.iter().map(|path| path.last().unwrap().fee_msat).sum();
 +                                                      if retry_amt_msat + *pending_amt_msat > *total_msat * (100 + RETRY_OVERFLOW_PERCENTAGE) / 100 {
 +                                                              return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 +                                                                      err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
 +                                                              }))
 +                                                      }
 +                                                      (*total_msat, *payment_hash, *payment_secret)
 +                                              },
 +                                              PendingOutboundPayment::Legacy { .. } => {
                                                        return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 -                                                              err: format!("retry_amt_msat of {} will put pending_amt_msat (currently: {}) more than 10% over total_payment_amt_msat of {}", retry_amt_msat, pending_amt_msat, total_msat).to_string()
 +                                                              err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
                                                        }))
 -                                              }
 -                                              (*total_msat, *payment_hash, *payment_secret)
 -                                      },
 -                                      PendingOutboundPayment::Legacy { .. } => {
 -                                              return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 -                                                      err: "Unable to retry payments that were initially sent on LDK versions prior to 0.0.102".to_string()
 -                                              }))
 -                                      },
 -                                      PendingOutboundPayment::Fulfilled { .. } => {
 -                                              return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 -                                                      err: "Payment already completed".to_owned()
 -                                              }));
 -                                      },
 -                                      PendingOutboundPayment::Abandoned { .. } => {
 -                                              return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 -                                                      err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
 -                                              }));
 -                                      },
 -                              }
 -                      } else {
 -                              return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 -                                      err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
 -                              }))
 +                                              },
 +                                              PendingOutboundPayment::Fulfilled { .. } => {
 +                                                      return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 +                                                              err: "Payment already completed".to_owned()
 +                                                      }));
 +                                              },
 +                                              PendingOutboundPayment::Abandoned { .. } => {
 +                                                      return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 +                                                              err: "Payment already abandoned (with some HTLCs still pending)".to_owned()
 +                                                      }));
 +                                              },
 +                                      };
 +                                      for (path, session_priv_bytes) in route.paths.iter().zip(onion_session_privs.iter()) {
 +                                              assert!(payment.insert(*session_priv_bytes, path));
 +                                      }
 +                                      res
 +                              },
 +                              None =>
 +                                      return Err(PaymentSendFailure::ParameterError(APIError::APIMisuseError {
 +                                              err: format!("Payment with ID {} not found", log_bytes!(payment_id.0)),
 +                                      })),
                        }
                };
 -              return self.send_payment_internal(route, payment_hash, &payment_secret, None, Some(payment_id), Some(total_msat)).map(|_| ())
 +              self.send_payment_internal(route, payment_hash, &payment_secret, None, payment_id, Some(total_msat), onion_session_privs)
        }
  
        /// Signals that no further retries for the given payment will occur.
        /// would be able to guess -- otherwise, an intermediate node may claim the payment and it will
        /// never reach the recipient.
        ///
 -      /// See [`send_payment`] documentation for more details on the return value of this function.
 +      /// See [`send_payment`] documentation for more details on the return value of this function
 +      /// and idempotency guarantees provided by the [`PaymentId`] key.
        ///
        /// Similar to regular payments, you MUST NOT reuse a `payment_preimage` value. See
        /// [`send_payment`] for more information about the risks of duplicate preimage usage.
        /// Note that `route` must have exactly one path.
        ///
        /// [`send_payment`]: Self::send_payment
 -      pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>) -> Result<(PaymentHash, PaymentId), PaymentSendFailure> {
 +      pub fn send_spontaneous_payment(&self, route: &Route, payment_preimage: Option<PaymentPreimage>, payment_id: PaymentId) -> Result<PaymentHash, PaymentSendFailure> {
                let preimage = match payment_preimage {
                        Some(p) => p,
                        None => PaymentPreimage(self.keys_manager.get_secure_random_bytes()),
                };
                let payment_hash = PaymentHash(Sha256::hash(&preimage.0).into_inner());
 -              match self.send_payment_internal(route, payment_hash, &None, Some(preimage), None, None) {
 -                      Ok(payment_id) => Ok((payment_hash, payment_id)),
 +              let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
 +
 +              match self.send_payment_internal(route, payment_hash, &None, Some(preimage), payment_id, None, onion_session_privs) {
 +                      Ok(()) => Ok(payment_hash),
                        Err(e) => Err(e)
                }
        }
                }
  
                let route = Route { paths: vec![hops], payment_params: None };
 +              let onion_session_privs = self.add_new_pending_payment(payment_hash, None, payment_id, &route)?;
  
 -              match self.send_payment_internal(&route, payment_hash, &None, None, Some(payment_id), None) {
 -                      Ok(payment_id) => Ok((payment_hash, payment_id)),
 +              match self.send_payment_internal(&route, payment_hash, &None, None, payment_id, None, onion_session_privs) {
 +                      Ok(()) => Ok((payment_hash, payment_id)),
                        Err(e) => Err(e)
                }
        }
  
        /// Handles the generation of a funding transaction, optionally (for tests) with a function
        /// which checks the correctness of the funding transaction given the associated channel.
 -      fn funding_transaction_generated_intern<FundingOutput: Fn(&Channel<Signer>, &Transaction) -> Result<OutPoint, APIError>>(
 +      fn funding_transaction_generated_intern<FundingOutput: Fn(&Channel<<K::Target as KeysInterface>::Signer>, &Transaction) -> Result<OutPoint, APIError>>(
                &self, temporary_channel_id: &[u8; 32], _counterparty_node_id: &PublicKey, funding_transaction: Transaction, find_funding_output: FundingOutput
        ) -> Result<(), APIError> {
                let (chan, msg) = {
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                if short_chan_id != 0 {
 -                                      let forward_chan_id = match channel_state.short_to_chan_info.get(&short_chan_id) {
 -                                              Some((_cp_id, chan_id)) => chan_id.clone(),
 -                                              None => {
 +                                      macro_rules! forwarding_channel_not_found {
 +                                              () => {
                                                        for forward_info in pending_forwards.drain(..) {
                                                                match forward_info {
 -                                                                      HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                              routing, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
 -                                                                              prev_funding_outpoint } => {
 -                                                                                      macro_rules! failure_handler {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
 -                                                                                                      log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
 -
 -                                                                                                      let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 -                                                                                                              short_channel_id: prev_short_channel_id,
 -                                                                                                              outpoint: prev_funding_outpoint,
 -                                                                                                              htlc_id: prev_htlc_id,
 -                                                                                                              incoming_packet_shared_secret: incoming_shared_secret,
 -                                                                                                              phantom_shared_secret: $phantom_ss,
 -                                                                                                      });
 -
 -                                                                                                      let reason = if $next_hop_unknown {
 -                                                                                                              HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
 -                                                                                                      } else {
 -                                                                                                              HTLCDestination::FailedPayment{ payment_hash }
 -                                                                                                      };
 -
 -                                                                                                      failed_forwards.push((htlc_source, payment_hash,
 -                                                                                                              HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
 -                                                                                                              reason
 -                                                                                                      ));
 -                                                                                                      continue;
 -                                                                                              }
 +                                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
 +                                                                              forward_info: PendingHTLCInfo {
 +                                                                                      routing, incoming_shared_secret, payment_hash, outgoing_amt_msat,
 +                                                                                      outgoing_cltv_value, incoming_amt_msat: _
 +                                                                              }
 +                                                                      }) => {
 +                                                                              macro_rules! failure_handler {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr, $next_hop_unknown: expr) => {
 +                                                                                              log_info!(self.logger, "Failed to accept/forward incoming HTLC: {}", $msg);
 +
 +                                                                                              let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 +                                                                                                      short_channel_id: prev_short_channel_id,
 +                                                                                                      outpoint: prev_funding_outpoint,
 +                                                                                                      htlc_id: prev_htlc_id,
 +                                                                                                      incoming_packet_shared_secret: incoming_shared_secret,
 +                                                                                                      phantom_shared_secret: $phantom_ss,
 +                                                                                              });
 +
 +                                                                                              let reason = if $next_hop_unknown {
 +                                                                                                      HTLCDestination::UnknownNextHop { requested_forward_scid: short_chan_id }
 +                                                                                              } else {
 +                                                                                                      HTLCDestination::FailedPayment{ payment_hash }
 +                                                                                              };
 +
 +                                                                                              failed_forwards.push((htlc_source, payment_hash,
 +                                                                                                      HTLCFailReason::Reason { failure_code: $err_code, data: $err_data },
 +                                                                                                      reason
 +                                                                                              ));
 +                                                                                              continue;
                                                                                        }
 -                                                                                      macro_rules! fail_forward {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 -                                                                                                      {
 -                                                                                                              failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
 -                                                                                                      }
 +                                                                              }
 +                                                                              macro_rules! fail_forward {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 +                                                                                              {
 +                                                                                                      failure_handler!($msg, $err_code, $err_data, $phantom_ss, true);
                                                                                                }
                                                                                        }
 -                                                                                      macro_rules! failed_payment {
 -                                                                                              ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 -                                                                                                      {
 -                                                                                                              failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
 -                                                                                                      }
 +                                                                              }
 +                                                                              macro_rules! failed_payment {
 +                                                                                      ($msg: expr, $err_code: expr, $err_data: expr, $phantom_ss: expr) => {
 +                                                                                              {
 +                                                                                                      failure_handler!($msg, $err_code, $err_data, $phantom_ss, false);
                                                                                                }
                                                                                        }
 -                                                                                      if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
 -                                                                                              let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
 -                                                                                              if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id) {
 -                                                                                                      let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
 -                                                                                                      let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
 -                                                                                                              Ok(res) => res,
 -                                                                                                              Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
 -                                                                                                                      let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
 -                                                                                                                      // In this scenario, the phantom would have sent us an
 -                                                                                                                      // `update_fail_malformed_htlc`, meaning here we encrypt the error as
 -                                                                                                                      // if it came from us (the second-to-last hop) but contains the sha256
 -                                                                                                                      // of the onion.
 -                                                                                                                      failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
 -                                                                                                              },
 -                                                                                                              Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
 -                                                                                                                      failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
 -                                                                                                              },
 -                                                                                                      };
 -                                                                                                      match next_hop {
 -                                                                                                              onion_utils::Hop::Receive(hop_data) => {
 -                                                                                                                      match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value, Some(phantom_shared_secret)) {
 -                                                                                                                              Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
 -                                                                                                                              Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
 -                                                                                                                      }
 -                                                                                                              },
 -                                                                                                              _ => panic!(),
 -                                                                                                      }
 -                                                                                              } else {
 -                                                                                                      fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
 +                                                                              }
 +                                                                              if let PendingHTLCRouting::Forward { onion_packet, .. } = routing {
 +                                                                                      let phantom_secret_res = self.keys_manager.get_node_secret(Recipient::PhantomNode);
 +                                                                                      if phantom_secret_res.is_ok() && fake_scid::is_valid_phantom(&self.fake_scid_rand_bytes, short_chan_id, &self.genesis_hash) {
 +                                                                                              let phantom_shared_secret = SharedSecret::new(&onion_packet.public_key.unwrap(), &phantom_secret_res.unwrap()).secret_bytes();
 +                                                                                              let next_hop = match onion_utils::decode_next_payment_hop(phantom_shared_secret, &onion_packet.hop_data, onion_packet.hmac, payment_hash) {
 +                                                                                                      Ok(res) => res,
 +                                                                                                      Err(onion_utils::OnionDecodeErr::Malformed { err_msg, err_code }) => {
 +                                                                                                              let sha256_of_onion = Sha256::hash(&onion_packet.hop_data).into_inner();
 +                                                                                                              // In this scenario, the phantom would have sent us an
 +                                                                                                              // `update_fail_malformed_htlc`, meaning here we encrypt the error as
 +                                                                                                              // if it came from us (the second-to-last hop) but contains the sha256
 +                                                                                                              // of the onion.
 +                                                                                                              failed_payment!(err_msg, err_code, sha256_of_onion.to_vec(), None);
 +                                                                                                      },
 +                                                                                                      Err(onion_utils::OnionDecodeErr::Relay { err_msg, err_code }) => {
 +                                                                                                              failed_payment!(err_msg, err_code, Vec::new(), Some(phantom_shared_secret));
 +                                                                                                      },
 +                                                                                              };
 +                                                                                              match next_hop {
 +                                                                                                      onion_utils::Hop::Receive(hop_data) => {
 +                                                                                                              match self.construct_recv_pending_htlc_info(hop_data, incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value, Some(phantom_shared_secret)) {
 +                                                                                                                      Ok(info) => phantom_receives.push((prev_short_channel_id, prev_funding_outpoint, vec![(info, prev_htlc_id)])),
 +                                                                                                                      Err(ReceiveError { err_code, err_data, msg }) => failed_payment!(msg, err_code, err_data, Some(phantom_shared_secret))
 +                                                                                                              }
 +                                                                                                      },
 +                                                                                                      _ => panic!(),
                                                                                                }
                                                                                        } else {
                                                                                                fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
                                                                                        }
 -                                                                              },
 +                                                                              } else {
 +                                                                                      fail_forward!(format!("Unknown short channel id {} for forward HTLC", short_chan_id), 0x4000 | 10, Vec::new(), None);
 +                                                                              }
 +                                                                      },
                                                                        HTLCForwardInfo::FailHTLC { .. } => {
                                                                                // Channel went away before we could fail it. This implies
                                                                                // the channel is now on chain and our counterparty is
                                                                        }
                                                                }
                                                        }
 +                                              }
 +                                      }
 +                                      let forward_chan_id = match self.short_to_chan_info.read().unwrap().get(&short_chan_id) {
 +                                              Some((_cp_id, chan_id)) => chan_id.clone(),
 +                                              None => {
 +                                                      forwarding_channel_not_found!();
                                                        continue;
                                                }
                                        };
 -                                      if let hash_map::Entry::Occupied(mut chan) = channel_state.by_id.entry(forward_chan_id) {
 -                                              let mut add_htlc_msgs = Vec::new();
 -                                              let mut fail_htlc_msgs = Vec::new();
 -                                              for forward_info in pending_forwards.drain(..) {
 -                                                      match forward_info {
 -                                                              HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                              routing: PendingHTLCRouting::Forward {
 -                                                                                      onion_packet, ..
 -                                                                              }, incoming_shared_secret, payment_hash, amt_to_forward, outgoing_cltv_value },
 -                                                                              prev_funding_outpoint } => {
 -                                                                      log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
 -                                                                      let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 -                                                                              short_channel_id: prev_short_channel_id,
 -                                                                              outpoint: prev_funding_outpoint,
 -                                                                              htlc_id: prev_htlc_id,
 -                                                                              incoming_packet_shared_secret: incoming_shared_secret,
 -                                                                              // Phantom payments are only PendingHTLCRouting::Receive.
 -                                                                              phantom_shared_secret: None,
 -                                                                      });
 -                                                                      match chan.get_mut().send_htlc(amt_to_forward, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
 -                                                                              Err(e) => {
 -                                                                                      if let ChannelError::Ignore(msg) = e {
 -                                                                                              log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
 -                                                                                      } else {
 -                                                                                              panic!("Stated return value requirements in send_htlc() were not met");
 -                                                                                      }
 -                                                                                      let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
 -                                                                                      failed_forwards.push((htlc_source, payment_hash,
 -                                                                                              HTLCFailReason::Reason { failure_code, data },
 -                                                                                              HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
 -                                                                                      ));
 -                                                                                      continue;
 +                                      match channel_state.by_id.entry(forward_chan_id) {
 +                                              hash_map::Entry::Vacant(_) => {
 +                                                      forwarding_channel_not_found!();
 +                                                      continue;
 +                                              },
 +                                              hash_map::Entry::Occupied(mut chan) => {
 +                                                      let mut add_htlc_msgs = Vec::new();
 +                                                      let mut fail_htlc_msgs = Vec::new();
 +                                                      for forward_info in pending_forwards.drain(..) {
 +                                                              match forward_info {
 +                                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint ,
 +                                                                              forward_info: PendingHTLCInfo {
 +                                                                                      incoming_shared_secret, payment_hash, outgoing_amt_msat, outgoing_cltv_value,
 +                                                                                      routing: PendingHTLCRouting::Forward { onion_packet, .. }, incoming_amt_msat: _,
                                                                                },
 -                                                                              Ok(update_add) => {
 -                                                                                      match update_add {
 -                                                                                              Some(msg) => { add_htlc_msgs.push(msg); },
 -                                                                                              None => {
 -                                                                                                      // Nothing to do here...we're waiting on a remote
 -                                                                                                      // revoke_and_ack before we can add anymore HTLCs. The Channel
 -                                                                                                      // will automatically handle building the update_add_htlc and
 -                                                                                                      // commitment_signed messages when we can.
 -                                                                                                      // TODO: Do some kind of timer to set the channel as !is_live()
 -                                                                                                      // as we don't really want others relying on us relaying through
 -                                                                                                      // this channel currently :/.
 +                                                                      }) => {
 +                                                                              log_trace!(self.logger, "Adding HTLC from short id {} with payment_hash {} to channel with short id {} after delay", prev_short_channel_id, log_bytes!(payment_hash.0), short_chan_id);
 +                                                                              let htlc_source = HTLCSource::PreviousHopData(HTLCPreviousHopData {
 +                                                                                      short_channel_id: prev_short_channel_id,
 +                                                                                      outpoint: prev_funding_outpoint,
 +                                                                                      htlc_id: prev_htlc_id,
 +                                                                                      incoming_packet_shared_secret: incoming_shared_secret,
 +                                                                                      // Phantom payments are only PendingHTLCRouting::Receive.
 +                                                                                      phantom_shared_secret: None,
 +                                                                              });
 +                                                                              match chan.get_mut().send_htlc(outgoing_amt_msat, payment_hash, outgoing_cltv_value, htlc_source.clone(), onion_packet, &self.logger) {
 +                                                                                      Err(e) => {
 +                                                                                              if let ChannelError::Ignore(msg) = e {
 +                                                                                                      log_trace!(self.logger, "Failed to forward HTLC with payment_hash {}: {}", log_bytes!(payment_hash.0), msg);
 +                                                                                              } else {
 +                                                                                                      panic!("Stated return value requirements in send_htlc() were not met");
 +                                                                                              }
 +                                                                                              let (failure_code, data) = self.get_htlc_temp_fail_err_and_data(0x1000|7, short_chan_id, chan.get());
 +                                                                                              failed_forwards.push((htlc_source, payment_hash,
 +                                                                                                      HTLCFailReason::Reason { failure_code, data },
 +                                                                                                      HTLCDestination::NextHopChannel { node_id: Some(chan.get().get_counterparty_node_id()), channel_id: forward_chan_id }
 +                                                                                              ));
 +                                                                                              continue;
 +                                                                                      },
 +                                                                                      Ok(update_add) => {
 +                                                                                              match update_add {
 +                                                                                                      Some(msg) => { add_htlc_msgs.push(msg); },
 +                                                                                                      None => {
 +                                                                                                              // Nothing to do here...we're waiting on a remote
 +                                                                                                              // revoke_and_ack before we can add anymore HTLCs. The Channel
 +                                                                                                              // will automatically handle building the update_add_htlc and
 +                                                                                                              // commitment_signed messages when we can.
 +                                                                                                              // TODO: Do some kind of timer to set the channel as !is_live()
 +                                                                                                              // as we don't really want others relying on us relaying through
 +                                                                                                              // this channel currently :/.
 +                                                                                                      }
                                                                                                }
                                                                                        }
                                                                                }
 -                                                                      }
 -                                                              },
 -                                                              HTLCForwardInfo::AddHTLC { .. } => {
 -                                                                      panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
 -                                                              },
 -                                                              HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
 -                                                                      log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
 -                                                                      match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
 -                                                                              Err(e) => {
 -                                                                                      if let ChannelError::Ignore(msg) = e {
 -                                                                                              log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
 -                                                                                      } else {
 -                                                                                              panic!("Stated return value requirements in get_update_fail_htlc() were not met");
 +                                                                      },
 +                                                                      HTLCForwardInfo::AddHTLC { .. } => {
 +                                                                              panic!("short_channel_id != 0 should imply any pending_forward entries are of type Forward");
 +                                                                      },
 +                                                                      HTLCForwardInfo::FailHTLC { htlc_id, err_packet } => {
 +                                                                              log_trace!(self.logger, "Failing HTLC back to channel with short id {} (backward HTLC ID {}) after delay", short_chan_id, htlc_id);
 +                                                                              match chan.get_mut().get_update_fail_htlc(htlc_id, err_packet, &self.logger) {
 +                                                                                      Err(e) => {
 +                                                                                              if let ChannelError::Ignore(msg) = e {
 +                                                                                                      log_trace!(self.logger, "Failed to fail HTLC with ID {} backwards to short_id {}: {}", htlc_id, short_chan_id, msg);
 +                                                                                              } else {
 +                                                                                                      panic!("Stated return value requirements in get_update_fail_htlc() were not met");
 +                                                                                              }
 +                                                                                              // fail-backs are best-effort, we probably already have one
 +                                                                                              // pending, and if not that's OK, if not, the channel is on
 +                                                                                              // the chain and sending the HTLC-Timeout is their problem.
 +                                                                                              continue;
 +                                                                                      },
 +                                                                                      Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
 +                                                                                      Ok(None) => {
 +                                                                                              // Nothing to do here...we're waiting on a remote
 +                                                                                              // revoke_and_ack before we can update the commitment
 +                                                                                              // transaction. The Channel will automatically handle
 +                                                                                              // building the update_fail_htlc and commitment_signed
 +                                                                                              // messages when we can.
 +                                                                                              // We don't need any kind of timer here as they should fail
 +                                                                                              // the channel onto the chain if they can't get our
 +                                                                                              // update_fail_htlc in time, it's not our problem.
                                                                                        }
 -                                                                                      // fail-backs are best-effort, we probably already have one
 -                                                                                      // pending, and if not that's OK, if not, the channel is on
 -                                                                                      // the chain and sending the HTLC-Timeout is their problem.
 -                                                                                      continue;
 -                                                                              },
 -                                                                              Ok(Some(msg)) => { fail_htlc_msgs.push(msg); },
 -                                                                              Ok(None) => {
 -                                                                                      // Nothing to do here...we're waiting on a remote
 -                                                                                      // revoke_and_ack before we can update the commitment
 -                                                                                      // transaction. The Channel will automatically handle
 -                                                                                      // building the update_fail_htlc and commitment_signed
 -                                                                                      // messages when we can.
 -                                                                                      // We don't need any kind of timer here as they should fail
 -                                                                                      // the channel onto the chain if they can't get our
 -                                                                                      // update_fail_htlc in time, it's not our problem.
                                                                                }
 -                                                                      }
 -                                                              },
 +                                                                      },
 +                                                              }
                                                        }
 -                                              }
  
 -                                              if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
 -                                                      let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
 -                                                              Ok(res) => res,
 -                                                              Err(e) => {
 -                                                                      // We surely failed send_commitment due to bad keys, in that case
 -                                                                      // close channel and then send error message to peer.
 -                                                                      let counterparty_node_id = chan.get().get_counterparty_node_id();
 -                                                                      let err: Result<(), _>  = match e {
 -                                                                              ChannelError::Ignore(_) | ChannelError::Warn(_) => {
 -                                                                                      panic!("Stated return value requirements in send_commitment() were not met");
 -                                                                              }
 -                                                                              ChannelError::Close(msg) => {
 -                                                                                      log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
 -                                                                                      let mut channel = remove_channel!(self, channel_state, chan);
 -                                                                                      // ChannelClosed event is generated by handle_error for us.
 -                                                                                      Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
 -                                                                              },
 -                                                                      };
 -                                                                      handle_errors.push((counterparty_node_id, err));
 -                                                                      continue;
 -                                                              }
 -                                                      };
 -                                                      match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
 -                                                              ChannelMonitorUpdateStatus::Completed => {},
 -                                                              e => {
 -                                                                      handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
 -                                                                      continue;
 +                                                      if !add_htlc_msgs.is_empty() || !fail_htlc_msgs.is_empty() {
 +                                                              let (commitment_msg, monitor_update) = match chan.get_mut().send_commitment(&self.logger) {
 +                                                                      Ok(res) => res,
 +                                                                      Err(e) => {
 +                                                                              // We surely failed send_commitment due to bad keys, in that case
 +                                                                              // close channel and then send error message to peer.
 +                                                                              let counterparty_node_id = chan.get().get_counterparty_node_id();
 +                                                                              let err: Result<(), _>  = match e {
 +                                                                                      ChannelError::Ignore(_) | ChannelError::Warn(_) => {
 +                                                                                              panic!("Stated return value requirements in send_commitment() were not met");
 +                                                                                      }
 +                                                                                      ChannelError::Close(msg) => {
 +                                                                                              log_trace!(self.logger, "Closing channel {} due to Close-required error: {}", log_bytes!(chan.key()[..]), msg);
 +                                                                                              let mut channel = remove_channel!(self, chan);
 +                                                                                              // ChannelClosed event is generated by handle_error for us.
 +                                                                                              Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel.channel_id(), channel.get_user_id(), channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
 +                                                                                      },
 +                                                                              };
 +                                                                              handle_errors.push((counterparty_node_id, err));
 +                                                                              continue;
 +                                                                      }
 +                                                              };
 +                                                              match self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update) {
 +                                                                      ChannelMonitorUpdateStatus::Completed => {},
 +                                                                      e => {
 +                                                                              handle_errors.push((chan.get().get_counterparty_node_id(), handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, true)));
 +                                                                              continue;
 +                                                                      }
                                                                }
 +                                                              log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
 +                                                                      add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
 +                                                              channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
 +                                                                      node_id: chan.get().get_counterparty_node_id(),
 +                                                                      updates: msgs::CommitmentUpdate {
 +                                                                              update_add_htlcs: add_htlc_msgs,
 +                                                                              update_fulfill_htlcs: Vec::new(),
 +                                                                              update_fail_htlcs: fail_htlc_msgs,
 +                                                                              update_fail_malformed_htlcs: Vec::new(),
 +                                                                              update_fee: None,
 +                                                                              commitment_signed: commitment_msg,
 +                                                                      },
 +                                                              });
                                                        }
 -                                                      log_debug!(self.logger, "Forwarding HTLCs resulted in a commitment update with {} HTLCs added and {} HTLCs failed for channel {}",
 -                                                              add_htlc_msgs.len(), fail_htlc_msgs.len(), log_bytes!(chan.get().channel_id()));
 -                                                      channel_state.pending_msg_events.push(events::MessageSendEvent::UpdateHTLCs {
 -                                                              node_id: chan.get().get_counterparty_node_id(),
 -                                                              updates: msgs::CommitmentUpdate {
 -                                                                      update_add_htlcs: add_htlc_msgs,
 -                                                                      update_fulfill_htlcs: Vec::new(),
 -                                                                      update_fail_htlcs: fail_htlc_msgs,
 -                                                                      update_fail_malformed_htlcs: Vec::new(),
 -                                                                      update_fee: None,
 -                                                                      commitment_signed: commitment_msg,
 -                                                              },
 -                                                      });
                                                }
 -                                      } else {
 -                                              unreachable!();
                                        }
                                } else {
                                        for forward_info in pending_forwards.drain(..) {
                                                match forward_info {
 -                                                      HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_htlc_id, forward_info: PendingHTLCInfo {
 -                                                                      routing, incoming_shared_secret, payment_hash, amt_to_forward, .. },
 -                                                                      prev_funding_outpoint } => {
 +                                                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_htlc_id, prev_funding_outpoint,
 +                                                              forward_info: PendingHTLCInfo {
 +                                                                      routing, incoming_shared_secret, payment_hash, outgoing_amt_msat, ..
 +                                                              }
 +                                                      }) => {
                                                                let (cltv_expiry, onion_payload, payment_data, phantom_shared_secret) = match routing {
                                                                        PendingHTLCRouting::Receive { payment_data, incoming_cltv_expiry, phantom_shared_secret } => {
                                                                                let _legacy_hop_data = Some(payment_data.clone());
                                                                                incoming_packet_shared_secret: incoming_shared_secret,
                                                                                phantom_shared_secret,
                                                                        },
 -                                                                      value: amt_to_forward,
 +                                                                      value: outgoing_amt_msat,
                                                                        timer_ticks: 0,
 -                                                                      total_msat: if let Some(data) = &payment_data { data.total_msat } else { amt_to_forward },
 +                                                                      total_msat: if let Some(data) = &payment_data { data.total_msat } else { outgoing_amt_msat },
                                                                        cltv_expiry,
                                                                        onion_payload,
                                                                };
                                                                                                                e.insert((purpose.clone(), vec![claimable_htlc]));
                                                                                                                new_events.push(events::Event::PaymentReceived {
                                                                                                                        payment_hash,
 -                                                                                                                      amount_msat: amt_to_forward,
 +                                                                                                                      amount_msat: outgoing_amt_msat,
                                                                                                                        purpose,
                                                                                                                });
                                                                                                        },
                self.process_background_events();
        }
  
 -      fn update_channel_fee(&self, short_to_chan_info: &mut HashMap<u64, (PublicKey, [u8; 32])>, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
 +      fn update_channel_fee(&self, pending_msg_events: &mut Vec<events::MessageSendEvent>, chan_id: &[u8; 32], chan: &mut Channel<<K::Target as KeysInterface>::Signer>, new_feerate: u32) -> (bool, NotifyOption, Result<(), MsgHandleErrInternal>) {
                if !chan.is_outbound() { return (true, NotifyOption::SkipPersist, Ok(())); }
                // If the feerate has decreased by less than half, don't bother
                if new_feerate <= chan.get_feerate() && new_feerate * 2 > chan.get_feerate() {
                let res = match chan.send_update_fee_and_commit(new_feerate, &self.logger) {
                        Ok(res) => Ok(res),
                        Err(e) => {
 -                              let (drop, res) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
 +                              let (drop, res) = convert_chan_err!(self, e, chan, chan_id);
                                if drop { retain_channel = false; }
                                Err(res)
                        }
                                                Ok(())
                                        },
                                        e => {
 -                                              let (res, drop) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
 +                                              let (res, drop) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, chan_id, COMMITMENT_UPDATE_ONLY);
                                                if drop { retain_channel = false; }
                                                res
                                        }
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
 -                              let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
 -                                      let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
 +                                      let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push(err);
                });
        }
  
 +      fn remove_stale_resolved_payments(&self) {
 +              // If an outbound payment was completed, and no pending HTLCs remain, we should remove it
 +              // from the map. However, if we did that immediately when the last payment HTLC is claimed,
 +              // this could race the user making a duplicate send_payment call and our idempotency
 +              // guarantees would be violated. Instead, we wait a few timer ticks to do the actual
 +              // removal. This should be more than sufficient to ensure the idempotency of any
 +              // `send_payment` calls that were made at the same time the `PaymentSent` event was being
 +              // processed.
 +              let mut pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
 +              let pending_events = self.pending_events.lock().unwrap();
 +              pending_outbound_payments.retain(|payment_id, payment| {
 +                      if let PendingOutboundPayment::Fulfilled { session_privs, timer_ticks_without_htlcs, .. } = payment {
 +                              let mut no_remaining_entries = session_privs.is_empty();
 +                              if no_remaining_entries {
 +                                      for ev in pending_events.iter() {
 +                                              match ev {
 +                                                      events::Event::PaymentSent { payment_id: Some(ev_payment_id), .. } |
 +                                                      events::Event::PaymentPathSuccessful { payment_id: ev_payment_id, .. } |
 +                                                      events::Event::PaymentPathFailed { payment_id: Some(ev_payment_id), .. } => {
 +                                                              if payment_id == ev_payment_id {
 +                                                                      no_remaining_entries = false;
 +                                                                      break;
 +                                                              }
 +                                                      },
 +                                                      _ => {},
 +                                              }
 +                                      }
 +                              }
 +                              if no_remaining_entries {
 +                                      *timer_ticks_without_htlcs += 1;
 +                                      *timer_ticks_without_htlcs <= IDEMPOTENCY_TIMEOUT_TICKS
 +                              } else {
 +                                      *timer_ticks_without_htlcs = 0;
 +                                      true
 +                              }
 +                      } else { true }
 +              });
 +      }
 +
        /// Performs actions which should happen on startup and roughly once per minute thereafter.
        ///
        /// This currently includes:
                                let mut channel_state_lock = self.channel_state.lock().unwrap();
                                let channel_state = &mut *channel_state_lock;
                                let pending_msg_events = &mut channel_state.pending_msg_events;
 -                              let short_to_chan_info = &mut channel_state.short_to_chan_info;
                                channel_state.by_id.retain(|chan_id, chan| {
                                        let counterparty_node_id = chan.get_counterparty_node_id();
 -                                      let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(short_to_chan_info, pending_msg_events, chan_id, chan, new_feerate);
 +                                      let (retain_channel, chan_needs_persist, err) = self.update_channel_fee(pending_msg_events, chan_id, chan, new_feerate);
                                        if chan_needs_persist == NotifyOption::DoPersist { should_persist = NotifyOption::DoPersist; }
                                        if err.is_err() {
                                                handle_errors.push((err, counterparty_node_id));
                                        if !retain_channel { return false; }
  
                                        if let Err(e) = chan.timer_check_closing_negotiation_progress() {
 -                                              let (needs_close, err) = convert_chan_err!(self, e, short_to_chan_info, chan, chan_id);
 +                                              let (needs_close, err) = convert_chan_err!(self, e, chan, chan_id);
                                                handle_errors.push((Err(err), chan.get_counterparty_node_id()));
                                                if needs_close { return false; }
                                        }
                        for (err, counterparty_node_id) in handle_errors.drain(..) {
                                let _ = handle_error!(self, err, counterparty_node_id);
                        }
 +
 +                      self.remove_stale_resolved_payments();
 +
                        should_persist
                });
        }
        ///
        /// This is for failures on the channel on which the HTLC was *received*, not failures
        /// forwarding
 -      fn get_htlc_inbound_temp_fail_err_and_data(&self, desired_err_code: u16, chan: &Channel<Signer>) -> (u16, Vec<u8>) {
 +      fn get_htlc_inbound_temp_fail_err_and_data(&self, desired_err_code: u16, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> (u16, Vec<u8>) {
                // We can't be sure what SCID was used when relaying inbound towards us, so we have to
                // guess somewhat. If its a public channel, we figure best to just use the real SCID (as
                // we're not leaking that we have a channel with the counterparty), otherwise we try to use
  
        /// Gets an HTLC onion failure code and error data for an `UPDATE` error, given the error code
        /// that we want to return and a channel.
 -      fn get_htlc_temp_fail_err_and_data(&self, desired_err_code: u16, scid: u64, chan: &Channel<Signer>) -> (u16, Vec<u8>) {
 +      fn get_htlc_temp_fail_err_and_data(&self, desired_err_code: u16, scid: u64, chan: &Channel<<K::Target as KeysInterface>::Signer>) -> (u16, Vec<u8>) {
                debug_assert_eq!(desired_err_code & 0x1000, 0x1000);
                if let Ok(upd) = self.get_channel_update_for_onion(scid, chan) {
                        let mut enc = VecWriter(Vec::with_capacity(upd.serialized_length() + 6));
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        for htlc in sources.iter() {
 -                              if let None = channel_state.short_to_chan_info.get(&htlc.prev_hop.short_channel_id) {
 +                              let chan_id = match self.short_to_chan_info.read().unwrap().get(&htlc.prev_hop.short_channel_id) {
 +                                      Some((_cp_id, chan_id)) => chan_id.clone(),
 +                                      None => {
 +                                              valid_mpp = false;
 +                                              break;
 +                                      }
 +                              };
 +
 +                              if let None = channel_state.by_id.get(&chan_id) {
                                        valid_mpp = false;
                                        break;
                                }
 +
                                if expected_amt_msat.is_some() && expected_amt_msat != Some(htlc.total_msat) {
                                        log_error!(self.logger, "Somehow ended up with an MPP payment with different total amounts - this should not be reachable!");
                                        debug_assert!(false);
                }
        }
  
 -      fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
 +      fn claim_funds_from_hop(&self, channel_state_lock: &mut MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, prev_hop: HTLCPreviousHopData, payment_preimage: PaymentPreimage) -> ClaimFundsFromHop {
                //TODO: Delay the claimed_funds relaying just like we do outbound relay!
                let channel_state = &mut **channel_state_lock;
 -              let chan_id = match channel_state.short_to_chan_info.get(&prev_hop.short_channel_id) {
 +              let chan_id = match self.short_to_chan_info.read().unwrap().get(&prev_hop.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                return ClaimFundsFromHop::PrevHopForceClosed
                                                                        payment_preimage, e);
                                                                return ClaimFundsFromHop::MonitorUpdateFail(
                                                                        chan.get().get_counterparty_node_id(),
 -                                                                      handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
 +                                                                      handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, false, msgs.is_some()).unwrap_err(),
                                                                        Some(htlc_value_msat)
                                                                );
                                                        }
                                                },
                                        }
                                        let counterparty_node_id = chan.get().get_counterparty_node_id();
 -                                      let (drop, res) = convert_chan_err!(self, e, channel_state.short_to_chan_info, chan.get_mut(), &chan_id);
 +                                      let (drop, res) = convert_chan_err!(self, e, chan.get_mut(), &chan_id);
                                        if drop {
                                                chan.remove_entry();
                                        }
                                        return ClaimFundsFromHop::MonitorUpdateFail(counterparty_node_id, res, None);
                                },
                        }
 -              } else { unreachable!(); }
 +              } else { return ClaimFundsFromHop::PrevHopForceClosed }
        }
  
        fn finalize_claims(&self, mut sources: Vec<HTLCSource>) {
                                                        }
                                                );
                                        }
 -                                      if payment.get().remaining_parts() == 0 {
 -                                              payment.remove();
 -                                      }
                                }
                        }
                }
        }
  
 -      fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
 +      fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<<K::Target as KeysInterface>::Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool, next_channel_id: [u8; 32]) {
                match source {
                        HTLCSource::OutboundRoute { session_priv, payment_id, path, .. } => {
                                mem::drop(channel_state_lock);
                                                                }
                                                        );
                                                }
 -
 -                                              if payment.get().remaining_parts() == 0 {
 -                                                      payment.remove();
 -                                              }
                                        }
                                } else {
                                        log_trace!(self.logger, "Received duplicative fulfill for HTLC with payment_preimage {}", log_bytes!(payment_preimage.0));
                                                }
                                        };
                                        channel_state.pending_msg_events.push(send_msg_err_event);
 -                                      let _ = remove_channel!(self, channel_state, channel);
 +                                      let _ = remove_channel!(self, channel);
                                        return Err(APIError::APIMisuseError { err: "Please use accept_inbound_channel_from_trusted_peer_0conf to accept channels with zero confirmations.".to_owned() });
                                }
  
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
 -                                      try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), channel_state, chan);
 +                                      try_chan_entry!(self, chan.get_mut().accept_channel(&msg, &self.default_configuration.channel_handshake_limits, &their_features), chan);
                                        (chan.get().get_value_satoshis(), chan.get().get_funding_redeemscript().to_v0_p2wsh(), chan.get().get_user_id())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.temporary_channel_id));
                                        }
 -                                      (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), channel_state, chan), chan.remove())
 +                                      (try_chan_entry!(self, chan.get_mut().funding_created(msg, best_block, &self.logger), chan), chan.remove())
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.temporary_channel_id))
                        }
                                        msg: funding_msg,
                                });
                                if let Some(msg) = channel_ready {
 -                                      send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan, msg);
 +                                      send_channel_ready!(self, channel_state.pending_msg_events, chan, msg);
                                }
                                e.insert(chan);
                        }
                                        }
                                        let (monitor, funding_tx, channel_ready) = match chan.get_mut().funding_signed(&msg, best_block, &self.logger) {
                                                Ok(update) => update,
 -                                              Err(e) => try_chan_entry!(self, Err(e), channel_state, chan),
 +                                              Err(e) => try_chan_entry!(self, Err(e), chan),
                                        };
                                        match self.chain_monitor.watch_channel(chan.get().get_funding_txo().unwrap(), monitor) {
                                                ChannelMonitorUpdateStatus::Completed => {},
                                                e => {
 -                                                      let mut res = handle_monitor_update_res!(self, e, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
 +                                                      let mut res = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::RevokeAndACKFirst, channel_ready.is_some(), OPTIONALLY_RESEND_FUNDING_LOCKED);
                                                        if let Err(MsgHandleErrInternal { ref mut shutdown_finish, .. }) = res {
                                                                // We weren't able to watch the channel to begin with, so no updates should be made on
                                                                // it. Previously, full_stack_target found an (unreachable) panic when the
                                                },
                                        }
                                        if let Some(msg) = channel_ready {
 -                                              send_channel_ready!(channel_state.short_to_chan_info, channel_state.pending_msg_events, chan.get(), msg);
 +                                              send_channel_ready!(self, channel_state.pending_msg_events, chan.get(), msg);
                                        }
                                        funding_tx
                                },
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
                                let announcement_sigs_opt = try_chan_entry!(self, chan.get_mut().channel_ready(&msg, self.get_our_node_id(),
 -                                      self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), channel_state, chan);
 +                                      self.genesis_hash.clone(), &self.best_block.read().unwrap(), &self.logger), chan);
                                if let Some(announcement_sigs) = announcement_sigs_opt {
                                        log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(chan.get().channel_id()));
                                        channel_state.pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
                                                });
                                        }
                                }
 +
 +                              emit_channel_ready_event!(self, chan.get_mut());
 +
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                                                        if chan_entry.get().sent_shutdown() { " after we initiated shutdown" } else { "" });
                                        }
  
 -                                      let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), channel_state, chan_entry);
 +                                      let (shutdown, monitor_update, htlcs) = try_chan_entry!(self, chan_entry.get_mut().shutdown(&self.keys_manager, &their_features, &msg), chan_entry);
                                        dropped_htlcs = htlcs;
  
                                        // Update the monitor with the shutdown script if necessary.
                                        if let Some(monitor_update) = monitor_update {
                                                let update_res = self.chain_monitor.update_channel(chan_entry.get().get_funding_txo().unwrap(), monitor_update);
                                                let (result, is_permanent) =
 -                                                      handle_monitor_update_res!(self, update_res, channel_state.short_to_chan_info, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
 +                                                      handle_monitor_update_res!(self, update_res, chan_entry.get_mut(), RAACommitmentOrder::CommitmentFirst, chan_entry.key(), NO_UPDATE);
                                                if is_permanent {
 -                                                      remove_channel!(self, channel_state, chan_entry);
 +                                                      remove_channel!(self, chan_entry);
                                                        break result;
                                                }
                                        }
                                        if chan_entry.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
 -                                      let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), channel_state, chan_entry);
 +                                      let (closing_signed, tx) = try_chan_entry!(self, chan_entry.get_mut().closing_signed(&self.fee_estimator, &msg), chan_entry);
                                        if let Some(msg) = closing_signed {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendClosingSigned {
                                                        node_id: counterparty_node_id.clone(),
                                                // also implies there are no pending HTLCs left on the channel, so we can
                                                // fully delete it from tracking (the channel monitor is still around to
                                                // watch for old state broadcasts)!
 -                                              (tx, Some(remove_channel!(self, channel_state, chan_entry)))
 +                                              (tx, Some(remove_channel!(self, chan_entry)))
                                        } else { (tx, None) }
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
  
 -                              let create_pending_htlc_status = |chan: &Channel<Signer>, pending_forward_info: PendingHTLCStatus, error_code: u16| {
 +                              let create_pending_htlc_status = |chan: &Channel<<K::Target as KeysInterface>::Signer>, pending_forward_info: PendingHTLCStatus, error_code: u16| {
                                        // If the update_add is completely bogus, the call will Err and we will close,
                                        // but if we've sent a shutdown and they haven't acknowledged it yet, we just
                                        // want to reject the new HTLC and fail it backwards instead of forwarding.
                                                _ => pending_forward_info
                                        }
                                };
 -                              try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), channel_state, chan);
 +                              try_chan_entry!(self, chan.get_mut().update_add_htlc(&msg, pending_forward_info, create_pending_htlc_status, &self.logger), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
                                        if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                                return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                        }
 -                                      try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), channel_state, chan)
 +                                      try_chan_entry!(self, chan.get_mut().update_fulfill_htlc(&msg), chan)
                                },
                                hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                        }
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
 -                              try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), channel_state, chan);
 +                              try_chan_entry!(self, chan.get_mut().update_fail_htlc(&msg, HTLCFailReason::LightningError { err: msg.reason.clone() }), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
                                }
                                if (msg.failure_code & 0x8000) == 0 {
                                        let chan_err: ChannelError = ChannelError::Close("Got update_fail_malformed_htlc with BADONION not set".to_owned());
 -                                      try_chan_entry!(self, Err(chan_err), channel_state, chan);
 +                                      try_chan_entry!(self, Err(chan_err), chan);
                                }
 -                              try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), channel_state, chan);
 +                              try_chan_entry!(self, chan.get_mut().update_fail_malformed_htlc(&msg, HTLCFailReason::Reason { failure_code: msg.failure_code, data: Vec::new() }), chan);
                                Ok(())
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                                }
                                let (revoke_and_ack, commitment_signed, monitor_update) =
                                        match chan.get_mut().commitment_signed(&msg, &self.logger) {
 -                                              Err((None, e)) => try_chan_entry!(self, Err(e), channel_state, chan),
 +                                              Err((None, e)) => try_chan_entry!(self, Err(e), chan),
                                                Err((Some(update), e)) => {
                                                        assert!(chan.get().is_awaiting_monitor_update());
                                                        let _ = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), update);
 -                                                      try_chan_entry!(self, Err(e), channel_state, chan);
 +                                                      try_chan_entry!(self, Err(e), chan);
                                                        unreachable!();
                                                },
                                                Ok(res) => res
                                        };
                                let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), monitor_update);
 -                              if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
 +                              if let Err(e) = handle_monitor_update_res!(self, update_res, chan, RAACommitmentOrder::RevokeAndACKFirst, true, commitment_signed.is_some()) {
                                        return Err(e);
                                }
  
                                                        PendingHTLCRouting::ReceiveKeysend { .. } => 0,
                                        }) {
                                                hash_map::Entry::Occupied(mut entry) => {
 -                                                      entry.get_mut().push(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
 -                                                                                                      prev_htlc_id, forward_info });
 +                                                      entry.get_mut().push(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info }));
                                                },
                                                hash_map::Entry::Vacant(entry) => {
 -                                                      entry.insert(vec!(HTLCForwardInfo::AddHTLC { prev_short_channel_id, prev_funding_outpoint,
 -                                                                                                   prev_htlc_id, forward_info }));
 +                                                      entry.insert(vec!(HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
 +                                                              prev_short_channel_id, prev_funding_outpoint, prev_htlc_id, forward_info })));
                                                }
                                        }
                                }
                                        }
                                        let was_paused_for_mon_update = chan.get().is_awaiting_monitor_update();
                                        let raa_updates = break_chan_entry!(self,
 -                                              chan.get_mut().revoke_and_ack(&msg, &self.logger), channel_state, chan);
 +                                              chan.get_mut().revoke_and_ack(&msg, &self.logger), chan);
                                        htlcs_to_fail = raa_updates.holding_cell_failed_htlcs;
                                        let update_res = self.chain_monitor.update_channel(chan.get().get_funding_txo().unwrap(), raa_updates.monitor_update);
                                        if was_paused_for_mon_update {
                                                break Err(MsgHandleErrInternal::ignore_no_close("Existing pending monitor update prevented responses to RAA".to_owned()));
                                        }
                                        if update_res != ChannelMonitorUpdateStatus::Completed {
 -                                              if let Err(e) = handle_monitor_update_res!(self, update_res, channel_state, chan,
 +                                              if let Err(e) = handle_monitor_update_res!(self, update_res, chan,
                                                                RAACommitmentOrder::CommitmentFirst, false,
                                                                raa_updates.commitment_update.is_some(), false,
                                                                raa_updates.accepted_htlcs, raa_updates.failed_htlcs,
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Err(MsgHandleErrInternal::send_err_msg_no_close("Got a message for a channel from the wrong node!".to_owned(), msg.channel_id));
                                }
 -                              try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), channel_state, chan);
 +                              try_chan_entry!(self, chan.get_mut().update_fee(&self.fee_estimator, &msg), chan);
                        },
                        hash_map::Entry::Vacant(_) => return Err(MsgHandleErrInternal::send_err_msg_no_close("Failed to find corresponding channel".to_owned(), msg.channel_id))
                }
  
                                channel_state.pending_msg_events.push(events::MessageSendEvent::BroadcastChannelAnnouncement {
                                        msg: try_chan_entry!(self, chan.get_mut().announcement_signatures(
 -                                              self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), channel_state, chan),
 +                                              self.get_our_node_id(), self.genesis_hash.clone(), self.best_block.read().unwrap().height(), msg), chan),
                                        // Note that announcement_signatures fails if the channel cannot be announced,
                                        // so get_channel_update_for_broadcast will never fail by the time we get here.
                                        update_msg: self.get_channel_update_for_broadcast(chan.get()).unwrap(),
  
        /// Returns ShouldPersist if anything changed, otherwise either SkipPersist or an Err.
        fn internal_channel_update(&self, counterparty_node_id: &PublicKey, msg: &msgs::ChannelUpdate) -> Result<NotifyOption, MsgHandleErrInternal> {
 -              let mut channel_state_lock = self.channel_state.lock().unwrap();
 -              let channel_state = &mut *channel_state_lock;
 -              let chan_id = match channel_state.short_to_chan_info.get(&msg.contents.short_channel_id) {
 +              let chan_id = match self.short_to_chan_info.read().unwrap().get(&msg.contents.short_channel_id) {
                        Some((_cp_id, chan_id)) => chan_id.clone(),
                        None => {
                                // It's not a local channel
                                return Ok(NotifyOption::SkipPersist)
                        }
                };
 +              let mut channel_state_lock = self.channel_state.lock().unwrap();
 +              let channel_state = &mut *channel_state_lock;
                match channel_state.by_id.entry(chan_id) {
                        hash_map::Entry::Occupied(mut chan) => {
                                if chan.get().get_counterparty_node_id() != *counterparty_node_id {
                                        return Ok(NotifyOption::SkipPersist);
                                } else {
                                        log_debug!(self.logger, "Received channel_update for channel {}.", log_bytes!(chan_id));
 -                                      try_chan_entry!(self, chan.get_mut().channel_update(&msg), channel_state, chan);
 +                                      try_chan_entry!(self, chan.get_mut().channel_update(&msg), chan);
                                }
                        },
 -                      hash_map::Entry::Vacant(_) => unreachable!()
 +                      hash_map::Entry::Vacant(_) => return Ok(NotifyOption::SkipPersist)
                }
                Ok(NotifyOption::DoPersist)
        }
                                        // add-HTLCs on disconnect, we may be handed HTLCs to fail backwards here.
                                        let responses = try_chan_entry!(self, chan.get_mut().channel_reestablish(
                                                msg, &self.logger, self.our_network_pubkey.clone(), self.genesis_hash,
 -                                              &*self.best_block.read().unwrap()), channel_state, chan);
 +                                              &*self.best_block.read().unwrap()), chan);
                                        let mut channel_update = None;
                                        if let Some(msg) = responses.shutdown_msg {
                                                channel_state.pending_msg_events.push(events::MessageSendEvent::SendShutdown {
                                                let by_id = &mut channel_state.by_id;
                                                let pending_msg_events = &mut channel_state.pending_msg_events;
                                                if let hash_map::Entry::Occupied(chan_entry) = by_id.entry(funding_outpoint.to_channel_id()) {
 -                                                      let mut chan = remove_channel!(self, channel_state, chan_entry);
 +                                                      let mut chan = remove_channel!(self, chan_entry);
                                                        failed_channels.push(chan.force_shutdown(false));
                                                        if let Ok(update) = self.get_channel_update_for_broadcast(&chan) {
                                                                pending_msg_events.push(events::MessageSendEvent::BroadcastChannelUpdate {
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
 -                      let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
  
                        by_id.retain(|channel_id, chan| {
                                                                },
                                                                e => {
                                                                        has_monitor_update = true;
 -                                                                      let (res, close_channel) = handle_monitor_update_res!(self, e, short_to_chan_info, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
 +                                                                      let (res, close_channel) = handle_monitor_update_res!(self, e, chan, RAACommitmentOrder::CommitmentFirst, channel_id, COMMITMENT_UPDATE_ONLY);
                                                                        handle_errors.push((chan.get_counterparty_node_id(), res));
                                                                        if close_channel { return false; }
                                                                },
                                                true
                                        },
                                        Err(e) => {
 -                                              let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
 +                                              let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                // ChannelClosed event is generated by handle_error for us
                                                !close_channel
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let by_id = &mut channel_state.by_id;
 -                      let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
  
                        by_id.retain(|channel_id, chan| {
  
                                                        log_info!(self.logger, "Broadcasting {}", log_tx!(tx));
                                                        self.tx_broadcaster.broadcast_transaction(&tx);
 -                                                      update_maps_on_chan_removal!(self, short_to_chan_info, chan);
 +                                                      update_maps_on_chan_removal!(self, chan);
                                                        false
                                                } else { true }
                                        },
                                        Err(e) => {
                                                has_update = true;
 -                                              let (close_channel, res) = convert_chan_err!(self, e, short_to_chan_info, chan, channel_id);
 +                                              let (close_channel, res) = convert_chan_err!(self, e, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
                                                !close_channel
                                        }
        ///
        /// [phantom node payments]: crate::chain::keysinterface::PhantomKeysManager
        pub fn get_phantom_scid(&self) -> u64 {
 -              let mut channel_state = self.channel_state.lock().unwrap();
 -              let best_block = self.best_block.read().unwrap();
 +              let best_block_height = self.best_block.read().unwrap().height();
 +              let short_to_chan_info = self.short_to_chan_info.read().unwrap();
                loop {
 -                      let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block.height(), &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
 +                      let scid_candidate = fake_scid::Namespace::Phantom.get_fake_scid(best_block_height, &self.genesis_hash, &self.fake_scid_rand_bytes, &self.keys_manager);
                        // Ensure the generated scid doesn't conflict with a real channel.
 -                      match channel_state.short_to_chan_info.entry(scid_candidate) {
 -                              hash_map::Entry::Occupied(_) => continue,
 -                              hash_map::Entry::Vacant(_) => return scid_candidate
 +                      match short_to_chan_info.get(&scid_candidate) {
 +                              Some(_) => continue,
 +                              None => return scid_candidate
                        }
                }
        }
        #[cfg(any(test, fuzzing, feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                let events = core::cell::RefCell::new(Vec::new());
 -              let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
 +              let event_handler = |event: events::Event| events.borrow_mut().push(event);
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
        pub fn clear_pending_payments(&self) {
                self.pending_outbound_payments.lock().unwrap().clear()
        }
 +
 +      /// Processes any events asynchronously in the order they were generated since the last call
 +      /// using the given event handler.
 +      ///
 +      /// See the trait-level documentation of [`EventsProvider`] for requirements.
 +      pub async fn process_pending_events_async<Future: core::future::Future, H: Fn(Event) -> Future>(
 +              &self, handler: H
 +      ) {
 +              // We'll acquire our total consistency lock until the returned future completes so that
 +              // we can be sure no other persists happen while processing events.
 +              let _read_guard = self.total_consistency_lock.read().unwrap();
 +
 +              let mut result = NotifyOption::SkipPersist;
 +
 +              // TODO: This behavior should be documented. It's unintuitive that we query
 +              // ChannelMonitors when clearing other events.
 +              if self.process_pending_monitor_events() {
 +                      result = NotifyOption::DoPersist;
 +              }
 +
 +              let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
 +              if !pending_events.is_empty() {
 +                      result = NotifyOption::DoPersist;
 +              }
 +
 +              for event in pending_events {
 +                      handler(event).await;
 +              }
 +
 +              if result == NotifyOption::DoPersist {
 +                      self.persistence_notifier.notify();
 +              }
 +      }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSendEventsProvider for ChannelManager<Signer, M, T, K, F, L>
 -      where M::Target: chain::Watch<Signer>,
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> MessageSendEventsProvider for ChannelManager<M, T, K, F, L>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
                                L::Target: Logger,
  {
        }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> EventsProvider for ChannelManager<Signer, M, T, K, F, L>
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> EventsProvider for ChannelManager<M, T, K, F, L>
  where
 -      M::Target: chain::Watch<Signer>,
 +      M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
 -      K::Target: KeysInterface<Signer = Signer>,
 +      K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
  {
                                result = NotifyOption::DoPersist;
                        }
  
 -                      let mut pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
 +                      let pending_events = mem::replace(&mut *self.pending_events.lock().unwrap(), vec![]);
                        if !pending_events.is_empty() {
                                result = NotifyOption::DoPersist;
                        }
  
 -                      for event in pending_events.drain(..) {
 -                              handler.handle_event(&event);
 +                      for event in pending_events {
 +                              handler.handle_event(event);
                        }
  
                        result
        }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Listen for ChannelManager<Signer, M, T, K, F, L>
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Listen for ChannelManager<M, T, K, F, L>
  where
 -      M::Target: chain::Watch<Signer>,
 +      M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
 -      K::Target: KeysInterface<Signer = Signer>,
 +      K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
  {
        }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Confirm for ChannelManager<Signer, M, T, K, F, L>
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> chain::Confirm for ChannelManager<M, T, K, F, L>
  where
 -      M::Target: chain::Watch<Signer>,
 +      M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
 -      K::Target: KeysInterface<Signer = Signer>,
 +      K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
  {
                payment_secrets.retain(|_, inbound_payment| {
                        inbound_payment.expiry_time > header.time as u64
                });
 -
 -              let mut outbounds = self.pending_outbound_payments.lock().unwrap();
 -              let mut pending_events = self.pending_events.lock().unwrap();
 -              outbounds.retain(|payment_id, payment| {
 -                      if payment.remaining_parts() != 0 { return true }
 -                      if let PendingOutboundPayment::Retryable { starting_block_height, payment_hash, .. } = payment {
 -                              if *starting_block_height + PAYMENT_EXPIRY_BLOCKS <= height {
 -                                      log_info!(self.logger, "Timing out payment with id {} and hash {}", log_bytes!(payment_id.0), log_bytes!(payment_hash.0));
 -                                      pending_events.push(events::Event::PaymentFailed {
 -                                              payment_id: *payment_id, payment_hash: *payment_hash,
 -                                      });
 -                                      false
 -                              } else { true }
 -                      } else { true }
 -              });
        }
  
 -      fn get_relevant_txids(&self) -> Vec<Txid> {
 +      fn get_relevant_txids(&self) -> Vec<(Txid, Option<BlockHash>)> {
                let channel_state = self.channel_state.lock().unwrap();
 -              let mut res = Vec::with_capacity(channel_state.short_to_chan_info.len());
 +              let mut res = Vec::with_capacity(channel_state.by_id.len());
                for chan in channel_state.by_id.values() {
 -                      if let Some(funding_txo) = chan.get_funding_txo() {
 -                              res.push(funding_txo.txid);
 +                      if let (Some(funding_txo), block_hash) = (chan.get_funding_txo(), chan.get_funding_tx_confirmed_in()) {
 +                              res.push((funding_txo.txid, block_hash));
                        }
                }
                res
        }
  }
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<Signer, M, T, K, F, L>
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelManager<M, T, K, F, L>
  where
 -      M::Target: chain::Watch<Signer>,
 +      M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
        T::Target: BroadcasterInterface,
 -      K::Target: KeysInterface<Signer = Signer>,
 +      K::Target: KeysInterface,
        F::Target: FeeEstimator,
        L::Target: Logger,
  {
        /// Calls a function which handles an on-chain event (blocks dis/connected, transactions
        /// un/confirmed, etc) on each channel, handling any resulting errors or messages generated by
        /// the function.
 -      fn do_chain_event<FN: Fn(&mut Channel<Signer>) -> Result<(Option<msgs::ChannelReady>, Vec<(HTLCSource, PaymentHash)>, Option<msgs::AnnouncementSignatures>), ClosureReason>>
 +      fn do_chain_event<FN: Fn(&mut Channel<<K::Target as KeysInterface>::Signer>) -> Result<(Option<msgs::ChannelReady>, Vec<(HTLCSource, PaymentHash)>, Option<msgs::AnnouncementSignatures>), ClosureReason>>
                        (&self, height_opt: Option<u32>, f: FN) {
                // Note that we MUST NOT end up calling methods on self.chain_monitor here - we're called
                // during initialization prior to the chain_monitor being fully configured in some cases.
                {
                        let mut channel_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_lock;
 -                      let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
                        channel_state.by_id.retain(|_, channel| {
                                let res = f(channel);
                                                }, HTLCDestination::NextHopChannel { node_id: Some(channel.get_counterparty_node_id()), channel_id: channel.channel_id() }));
                                        }
                                        if let Some(channel_ready) = channel_ready_opt {
 -                                              send_channel_ready!(short_to_chan_info, pending_msg_events, channel, channel_ready);
 +                                              send_channel_ready!(self, pending_msg_events, channel, channel_ready);
                                                if channel.is_usable() {
                                                        log_trace!(self.logger, "Sending channel_ready with private initial channel_update for our counterparty on channel {}", log_bytes!(channel.channel_id()));
                                                        if let Ok(msg) = self.get_channel_update_for_unicast(channel) {
                                                        log_trace!(self.logger, "Sending channel_ready WITHOUT channel_update for {}", log_bytes!(channel.channel_id()));
                                                }
                                        }
 +
 +                                      emit_channel_ready_event!(self, channel);
 +
                                        if let Some(announcement_sigs) = announcement_sigs {
                                                log_trace!(self.logger, "Sending announcement_signatures for channel {}", log_bytes!(channel.channel_id()));
                                                pending_msg_events.push(events::MessageSendEvent::SendAnnouncementSignatures {
                                                        // enforce option_scid_alias then), and if the funding tx is ever
                                                        // un-confirmed we force-close the channel, ensuring short_to_chan_info
                                                        // is always consistent.
 +                                                      let mut short_to_chan_info = self.short_to_chan_info.write().unwrap();
                                                        let scid_insert = short_to_chan_info.insert(real_scid, (channel.get_counterparty_node_id(), channel.channel_id()));
                                                        assert!(scid_insert.is_none() || scid_insert.unwrap() == (channel.get_counterparty_node_id(), channel.channel_id()),
                                                                "SCIDs should never collide - ensure you weren't behind by a full {} blocks when creating channels",
                                                }
                                        }
                                } else if let Err(reason) = res {
 -                                      update_maps_on_chan_removal!(self, short_to_chan_info, channel);
 +                                      update_maps_on_chan_removal!(self, channel);
                                        // It looks like our counterparty went on-chain or funding transaction was
                                        // reorged out of the main chain. Close the channel.
                                        failed_channels.push(channel.force_shutdown(true));
        }
  }
  
 -impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
 -      ChannelMessageHandler for ChannelManager<Signer, M, T, K, F, L>
 -      where M::Target: chain::Watch<Signer>,
 +impl<M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
 +      ChannelMessageHandler for ChannelManager<M, T, K, F, L>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
                        let pending_msg_events = &mut channel_state.pending_msg_events;
 -                      let short_to_chan_info = &mut channel_state.short_to_chan_info;
                        log_debug!(self.logger, "Marking channels with {} disconnected and generating channel_updates. We believe we {} make future connections to this peer.",
                                log_pubkey!(counterparty_node_id), if no_connection_possible { "cannot" } else { "can" });
                        channel_state.by_id.retain(|_, chan| {
                                if chan.get_counterparty_node_id() == *counterparty_node_id {
                                        chan.remove_uncommitted_htlcs_and_mark_paused(&self.logger);
                                        if chan.is_shutdown() {
 -                                              update_maps_on_chan_removal!(self, short_to_chan_info, chan);
 +                                              update_maps_on_chan_removal!(self, chan);
                                                self.issue_channel_close_events(chan, ClosureReason::DisconnectedPeer);
                                                return false;
                                        } else {
@@@ -6505,9 -6315,8 +6497,9 @@@ impl_writeable_tlv_based!(PendingHTLCIn
        (0, routing, required),
        (2, incoming_shared_secret, required),
        (4, payment_hash, required),
 -      (6, amt_to_forward, required),
 -      (8, outgoing_cltv_value, required)
 +      (6, outgoing_amt_msat, required),
 +      (8, outgoing_cltv_value, required),
 +      (9, incoming_amt_msat, option),
  });
  
  
@@@ -6706,7 -6515,7 +6698,7 @@@ impl Writeable for HTLCSource 
                                        (1, payment_id_opt, option),
                                        (2, first_hop_htlc_msat, required),
                                        (3, payment_secret, option),
 -                                      (4, path, vec_type),
 +                                      (4, *path, vec_type),
                                        (5, payment_params, option),
                                 });
                        }
@@@ -6729,20 -6538,18 +6721,20 @@@ impl_writeable_tlv_based_enum!(HTLCFail
        },
  ;);
  
 +impl_writeable_tlv_based!(PendingAddHTLCInfo, {
 +      (0, forward_info, required),
 +      (2, prev_short_channel_id, required),
 +      (4, prev_htlc_id, required),
 +      (6, prev_funding_outpoint, required),
 +});
 +
  impl_writeable_tlv_based_enum!(HTLCForwardInfo,
 -      (0, AddHTLC) => {
 -              (0, forward_info, required),
 -              (2, prev_short_channel_id, required),
 -              (4, prev_htlc_id, required),
 -              (6, prev_funding_outpoint, required),
 -      },
        (1, FailHTLC) => {
                (0, htlc_id, required),
                (2, err_packet, required),
 -      },
 -;);
 +      };
 +      (0, AddHTLC)
 +);
  
  impl_writeable_tlv_based!(PendingInboundPayment, {
        (0, payment_secret, required),
@@@ -6759,7 -6566,6 +6751,7 @@@ impl_writeable_tlv_based_enum_upgradabl
        (1, Fulfilled) => {
                (0, session_privs, required),
                (1, payment_hash, option),
 +              (3, timer_ticks_without_htlcs, (default_value, 0)),
        },
        (2, Retryable) => {
                (0, session_privs, required),
        },
  );
  
 -impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelManager<Signer, M, T, K, F, L>
 -      where M::Target: chain::Watch<Signer>,
 +impl<M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable for ChannelManager<M, T, K, F, L>
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
@@@ -7025,29 -6831,29 +7017,29 @@@ impl<'a, Signer: 'a + Sign, M: Deref, T
  
  // Implement ReadableArgs for an Arc'd ChannelManager to make it a bit easier to work with the
  // SipmleArcChannelManager type:
 -impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      ReadableArgs<ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<Signer, M, T, K, F, L>>)
 -      where M::Target: chain::Watch<Signer>,
 +impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 +      ReadableArgs<ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>> for (BlockHash, Arc<ChannelManager<M, T, K, F, L>>)
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
 -      fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
 -              let (blockhash, chan_manager) = <(BlockHash, ChannelManager<Signer, M, T, K, F, L>)>::read(reader, args)?;
 +      fn read<R: io::Read>(reader: &mut R, args: ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
 +              let (blockhash, chan_manager) = <(BlockHash, ChannelManager<M, T, K, F, L>)>::read(reader, args)?;
                Ok((blockhash, Arc::new(chan_manager)))
        }
  }
  
 -impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 -      ReadableArgs<ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>> for (BlockHash, ChannelManager<Signer, M, T, K, F, L>)
 -      where M::Target: chain::Watch<Signer>,
 +impl<'a, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
 +      ReadableArgs<ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>> for (BlockHash, ChannelManager<M, T, K, F, L>)
 +      where M::Target: chain::Watch<<K::Target as KeysInterface>::Signer>,
          T::Target: BroadcasterInterface,
 -        K::Target: KeysInterface<Signer = Signer>,
 +        K::Target: KeysInterface,
          F::Target: FeeEstimator,
          L::Target: Logger,
  {
 -      fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
 +      fn read<R: io::Read>(reader: &mut R, mut args: ChannelManagerReadArgs<'a, <K::Target as KeysInterface>::Signer, M, T, K, F, L>) -> Result<Self, DecodeError> {
                let _ver = read_ver_prefix!(reader, SERIALIZATION_VERSION);
  
                let genesis_hash: BlockHash = Readable::read(reader)?;
                let mut short_to_chan_info = HashMap::with_capacity(cmp::min(channel_count as usize, 128));
                let mut channel_closures = Vec::new();
                for _ in 0..channel_count {
 -                      let mut channel: Channel<Signer> = Channel::read(reader, (&args.keys_manager, best_block_height))?;
 +                      let mut channel: Channel<<K::Target as KeysInterface>::Signer> = Channel::read(reader, (&args.keys_manager, best_block_height))?;
                        let funding_txo = channel.get_funding_txo().ok_or(DecodeError::InvalidValue)?;
                        funding_txo_set.insert(funding_txo.clone());
                        if let Some(ref mut monitor) = args.channel_monitors.get_mut(&funding_txo) {
  
                        channel_state: Mutex::new(ChannelHolder {
                                by_id,
 -                              short_to_chan_info,
                                claimable_htlcs,
                                pending_msg_events: Vec::new(),
                        }),
                        forward_htlcs: Mutex::new(forward_htlcs),
                        outbound_scid_aliases: Mutex::new(outbound_scid_aliases),
                        id_to_peer: Mutex::new(id_to_peer),
 +                      short_to_chan_info: FairRwLock::new(short_to_chan_info),
                        fake_scid_rand_bytes: fake_scid_rand_bytes.unwrap(),
  
                        probing_cookie_secret: probing_cookie_secret.unwrap(),
@@@ -7592,22 -7398,18 +7584,22 @@@ mod tests 
  
                // First, send a partial MPP payment.
                let (route, our_payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(&nodes[0], nodes[1], 100_000);
 +              let mut mpp_route = route.clone();
 +              mpp_route.paths.push(mpp_route.paths[0].clone());
 +
                let payment_id = PaymentId([42; 32]);
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
 -              nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None).unwrap();
 +              let session_privs = nodes[0].node.add_new_pending_payment(our_payment_hash, Some(payment_secret), payment_id, &mpp_route).unwrap();
 +              nodes[0].node.send_payment_along_path(&mpp_route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[0]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                pass_along_path(&nodes[0], &[&nodes[1]], 200_000, our_payment_hash, Some(payment_secret), events.drain(..).next().unwrap(), false, None);
  
                // Next, send a keysend payment with the same payment_hash and make sure it fails.
 -              nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
 +              nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                expect_payment_failed!(nodes[0], our_payment_hash, true);
  
                // Send the second half of the original MPP payment.
 -              nodes[0].node.send_payment_along_path(&route.paths[0], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None).unwrap();
 +              nodes[0].node.send_payment_along_path(&mpp_route.paths[1], &route.payment_params, &our_payment_hash, &Some(payment_secret), 200_000, cur_height, payment_id, &None, session_privs[1]).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
 -              nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
 +              nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                        &nodes[0].node.get_our_node_id(), &route_params, &nodes[0].network_graph,
                        None, nodes[0].logger, &scorer, &random_seed_bytes
                ).unwrap();
 -              let (payment_hash, _) = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
 +              let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage), PaymentId(payment_preimage.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
  
                // Next, attempt a regular payment and make sure it fails.
                let payment_secret = PaymentSecret([43; 32]);
 -              nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
 +              nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], channelmanager::provided_init_features(), channelmanager::provided_init_features());
                let route_params = RouteParameters {
                        payment_params: PaymentParameters::for_keysend(payee_pubkey),
 -                      final_value_msat: 10000,
 +                      final_value_msat: 10_000,
                        final_cltv_expiry_delta: 40,
                };
                let network_graph = nodes[0].network_graph;
  
                let test_preimage = PaymentPreimage([42; 32]);
                let mismatch_payment_hash = PaymentHash([43; 32]);
 -              let _ = nodes[0].node.send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), None, None).unwrap();
 +              let session_privs = nodes[0].node.add_new_pending_payment(mismatch_payment_hash, None, PaymentId(mismatch_payment_hash.0), &route).unwrap();
 +              nodes[0].node.send_payment_internal(&route, mismatch_payment_hash, &None, Some(test_preimage), PaymentId(mismatch_payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
  
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], channelmanager::provided_init_features(), channelmanager::provided_init_features());
                let route_params = RouteParameters {
                        payment_params: PaymentParameters::for_keysend(payee_pubkey),
 -                      final_value_msat: 10000,
 +                      final_value_msat: 10_000,
                        final_cltv_expiry_delta: 40,
                };
                let network_graph = nodes[0].network_graph;
                let test_preimage = PaymentPreimage([42; 32]);
                let test_secret = PaymentSecret([43; 32]);
                let payment_hash = PaymentHash(Sha256::hash(&test_preimage.0).into_inner());
 -              let _ = nodes[0].node.send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), None, None).unwrap();
 +              let session_privs = nodes[0].node.add_new_pending_payment(payment_hash, Some(test_secret), PaymentId(payment_hash.0), &route).unwrap();
 +              nodes[0].node.send_payment_internal(&route, payment_hash, &Some(test_secret), Some(test_preimage), PaymentId(payment_hash.0), None, session_privs).unwrap();
                check_added_monitors!(nodes[0], 1);
  
                let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
                route.paths[1][0].short_channel_id = chan_2_id;
                route.paths[1][1].short_channel_id = chan_4_id;
  
 -              match nodes[0].node.send_payment(&route, payment_hash, &None).unwrap_err() {
 +              match nodes[0].node.send_payment(&route, payment_hash, &None, PaymentId(payment_hash.0)).unwrap_err() {
                        PaymentSendFailure::ParameterError(APIError::APIMisuseError { ref err }) => {
                                assert!(regex::Regex::new(r"Payment secret is required for multi-path payments").unwrap().is_match(err))                        },
                        _ => panic!("unexpected error")
@@@ -8067,7 -7867,7 +8059,7 @@@ pub mod bench 
        use crate::chain::Listen;
        use crate::chain::chainmonitor::{ChainMonitor, Persist};
        use crate::chain::keysinterface::{KeysManager, KeysInterface, InMemorySigner};
 -      use crate::ln::channelmanager::{self, BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage};
 +      use crate::ln::channelmanager::{self, BestBlock, ChainParameters, ChannelManager, PaymentHash, PaymentPreimage, PaymentId};
        use crate::ln::functional_test_utils::*;
        use crate::ln::msgs::{ChannelMessageHandler, Init};
        use crate::routing::gossip::NetworkGraph;
        use test::Bencher;
  
        struct NodeHolder<'a, P: Persist<InMemorySigner>> {
 -              node: &'a ChannelManager<InMemorySigner,
 +              node: &'a ChannelManager<
                        &'a ChainMonitor<InMemorySigner, &'a test_utils::TestChainSource,
                                &'a test_utils::TestBroadcaster, &'a test_utils::TestFeeEstimator,
                                &'a test_utils::TestLogger, &'a P>,
                        &'a test_utils::TestBroadcaster, &'a KeysManager,
 -                      &'a test_utils::TestFeeEstimator, &'a test_utils::TestLogger>
 +                      &'a test_utils::TestFeeEstimator, &'a test_utils::TestLogger>,
        }
  
        #[cfg(test)]
                        _ => panic!(),
                }
  
 +              let events_a = node_a.get_and_clear_pending_events();
 +              assert_eq!(events_a.len(), 1);
 +              match events_a[0] {
 +                      Event::ChannelReady{ ref counterparty_node_id, .. } => {
 +                              assert_eq!(*counterparty_node_id, node_b.get_our_node_id());
 +                      },
 +                      _ => panic!("Unexpected event"),
 +              }
 +
 +              let events_b = node_b.get_and_clear_pending_events();
 +              assert_eq!(events_b.len(), 1);
 +              match events_b[0] {
 +                      Event::ChannelReady{ ref counterparty_node_id, .. } => {
 +                              assert_eq!(*counterparty_node_id, node_a.get_our_node_id());
 +                      },
 +                      _ => panic!("Unexpected event"),
 +              }
 +
                let dummy_graph = NetworkGraph::new(genesis_hash, &logger_a);
  
                let mut payment_count: u64 = 0;
                                let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
                                let payment_secret = $node_b.create_inbound_payment_for_hash(payment_hash, None, 7200).unwrap();
  
 -                              $node_a.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
 +                              $node_a.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
                                let payment_event = SendEvent::from_event($node_a.get_and_clear_pending_msg_events().pop().unwrap());
                                $node_b.handle_update_add_htlc(&$node_a.get_our_node_id(), &payment_event.msgs[0]);
                                $node_b.handle_commitment_signed(&$node_a.get_our_node_id(), &payment_event.commitment_msg);
diff --combined lightning/src/ln/msgs.rs
index 8b5a6ac6fd24e32f4ba2dcc25c3715eb86531b30,4a0b58937ea46cf7624f15b4eaba6fb564d5accb..1478f8b6bfb8272795dcd06d8272db926876a959
@@@ -42,7 -42,7 +42,7 @@@ use crate::io_extras::read_to_end
  
  use crate::util::events::{MessageSendEventsProvider, OnionMessageProvider};
  use crate::util::logger;
- use crate::util::ser::{BigSize, LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
+ use crate::util::ser::{LengthReadable, Readable, ReadableArgs, Writeable, Writer, FixedLengthReader, HighZeroBytesDroppedBigSize, Hostname};
  
  use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
  
@@@ -66,7 -66,8 +66,7 @@@ pub enum DecodeError 
        /// A length descriptor in the packet didn't describe the later data correctly
        BadLengthDescriptor,
        /// Error from std::io
 -      Io(/// (C-not exported) as ErrorKind doesn't have a reasonable mapping
 -        io::ErrorKind),
 +      Io(io::ErrorKind),
        /// The message included zlib-compressed values, which we don't support.
        UnsupportedCompression,
  }
@@@ -1027,9 -1028,6 +1027,6 @@@ mod fuzzy_internal_msgs 
        }
  
        pub(crate) enum OnionHopDataFormat {
-               Legacy { // aka Realm-0
-                       short_channel_id: u64,
-               },
                NonFinalNode {
                        short_channel_id: u64,
                },
                /// Message serialization may panic if this value is more than 21 million Bitcoin.
                pub(crate) amt_to_forward: u64,
                pub(crate) outgoing_cltv_value: u32,
-               // 12 bytes of 0-padding for Legacy format
        }
  
        pub struct DecodedOnionErrorPacket {
@@@ -1458,13 -1455,6 +1454,6 @@@ impl Readable for FinalOnionHopData 
  impl Writeable for OnionHopData {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                match self.format {
-                       OnionHopDataFormat::Legacy { short_channel_id } => {
-                               0u8.write(w)?;
-                               short_channel_id.write(w)?;
-                               self.amt_to_forward.write(w)?;
-                               self.outgoing_cltv_value.write(w)?;
-                               w.write_all(&[0;12])?;
-                       },
                        OnionHopDataFormat::NonFinalNode { short_channel_id } => {
                                encode_varint_length_prefixed_tlv!(w, {
                                        (2, HighZeroBytesDroppedBigSize(self.amt_to_forward), required),
  
  impl Readable for OnionHopData {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let b: BigSize = Readable::read(r)?;
-               const LEGACY_ONION_HOP_FLAG: u64 = 0;
-               let (format, amt, cltv_value) = if b.0 != LEGACY_ONION_HOP_FLAG {
-                       let mut rd = FixedLengthReader::new(r, b.0);
-                       let mut amt = HighZeroBytesDroppedBigSize(0u64);
-                       let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
-                       let mut short_id: Option<u64> = None;
-                       let mut payment_data: Option<FinalOnionHopData> = None;
-                       let mut keysend_preimage: Option<PaymentPreimage> = None;
-                       decode_tlv_stream!(&mut rd, {
-                               (2, amt, required),
-                               (4, cltv_value, required),
-                               (6, short_id, option),
-                               (8, payment_data, option),
-                               // See https://github.com/lightning/blips/blob/master/blip-0003.md
-                               (5482373484, keysend_preimage, option)
-                       });
-                       rd.eat_remaining().map_err(|_| DecodeError::ShortRead)?;
-                       let format = if let Some(short_channel_id) = short_id {
-                               if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
-                               OnionHopDataFormat::NonFinalNode {
-                                       short_channel_id,
-                               }
-                       } else {
-                               if let &Some(ref data) = &payment_data {
-                                       if data.total_msat > MAX_VALUE_MSAT {
-                                               return Err(DecodeError::InvalidValue);
-                                       }
-                               }
-                               OnionHopDataFormat::FinalNode {
-                                       payment_data,
-                                       keysend_preimage,
-                               }
-                       };
-                       (format, amt.0, cltv_value.0)
+               let mut amt = HighZeroBytesDroppedBigSize(0u64);
+               let mut cltv_value = HighZeroBytesDroppedBigSize(0u32);
+               let mut short_id: Option<u64> = None;
+               let mut payment_data: Option<FinalOnionHopData> = None;
+               let mut keysend_preimage: Option<PaymentPreimage> = None;
+               read_tlv_fields!(r, {
+                       (2, amt, required),
+                       (4, cltv_value, required),
+                       (6, short_id, option),
+                       (8, payment_data, option),
+                       // See https://github.com/lightning/blips/blob/master/blip-0003.md
+                       (5482373484, keysend_preimage, option)
+               });
+               let format = if let Some(short_channel_id) = short_id {
+                       if payment_data.is_some() { return Err(DecodeError::InvalidValue); }
+                       OnionHopDataFormat::NonFinalNode {
+                               short_channel_id,
+                       }
                } else {
-                       let format = OnionHopDataFormat::Legacy {
-                               short_channel_id: Readable::read(r)?,
-                       };
-                       let amt: u64 = Readable::read(r)?;
-                       let cltv_value: u32 = Readable::read(r)?;
-                       r.read_exact(&mut [0; 12])?;
-                       (format, amt, cltv_value)
+                       if let &Some(ref data) = &payment_data {
+                               if data.total_msat > MAX_VALUE_MSAT {
+                                       return Err(DecodeError::InvalidValue);
+                               }
+                       }
+                       OnionHopDataFormat::FinalNode {
+                               payment_data,
+                               keysend_preimage,
+                       }
                };
  
-               if amt > MAX_VALUE_MSAT {
+               if amt.0 > MAX_VALUE_MSAT {
                        return Err(DecodeError::InvalidValue);
                }
                Ok(OnionHopData {
                        format,
-                       amt_to_forward: amt,
-                       outgoing_cltv_value: cltv_value,
+                       amt_to_forward: amt.0,
+                       outgoing_cltv_value: cltv_value.0,
                })
        }
  }
@@@ -2668,20 -2644,6 +2643,6 @@@ mod tests 
                assert_eq!(encoded_value, target_value);
        }
  
-       #[test]
-       fn encoding_legacy_onion_hop_data() {
-               let msg = msgs::OnionHopData {
-                       format: OnionHopDataFormat::Legacy {
-                               short_channel_id: 0xdeadbeef1bad1dea,
-                       },
-                       amt_to_forward: 0x0badf00d01020304,
-                       outgoing_cltv_value: 0xffffffff,
-               };
-               let encoded_value = msg.encode();
-               let target_value = hex::decode("00deadbeef1bad1dea0badf00d01020304ffffffff000000000000000000000000").unwrap();
-               assert_eq!(encoded_value, target_value);
-       }
        #[test]
        fn encoding_nonfinal_onion_hop_data() {
                let mut msg = msgs::OnionHopData {
index 13bbe8cd166e0147a6b0bd2469517359633e232a,a26e68e728212054fafe3b67559e697063579b0f..b5c55ccf5ca0c60f945dc4b87c516c83ff27a6fa
@@@ -15,11 -15,11 +15,11 @@@ use crate::chain::channelmonitor::{Chan
  use crate::chain::keysinterface::{KeysInterface, Recipient};
  use crate::ln::{PaymentHash, PaymentSecret};
  use crate::ln::channel::EXPIRE_PREV_CONFIG_TICKS;
 -use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingHTLCInfo, PendingHTLCRouting};
 +use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, HTLCForwardInfo, CLTV_FAR_FAR_AWAY, MIN_CLTV_EXPIRY_DELTA, PendingAddHTLCInfo, PendingHTLCInfo, PendingHTLCRouting, PaymentId};
  use crate::ln::onion_utils;
- use crate::routing::gossip::{NetworkUpdate, RoutingFees, NodeId};
+ use crate::routing::gossip::{NetworkUpdate, RoutingFees};
  use crate::routing::router::{get_route, PaymentParameters, Route, RouteHint, RouteHintHop};
- use crate::ln::features::{InitFeatures, InvoiceFeatures, NodeFeatures};
+ use crate::ln::features::{InitFeatures, InvoiceFeatures};
  use crate::ln::msgs;
  use crate::ln::msgs::{ChannelMessageHandler, ChannelUpdate};
  use crate::ln::wire::Encode;
@@@ -82,8 -82,7 +82,8 @@@ fn run_onion_failure_test_with_fail_int
        }
  
        // 0 ~~> 2 send payment
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(*payment_secret)).unwrap();
 +      let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 +      nodes[0].node.send_payment(&route, *payment_hash, &Some(*payment_secret), payment_id).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        // temper update_add (0 => 1)
        } else {
                panic!("Unexpected event");
        }
 +      nodes[0].node.abandon_payment(payment_id);
 +      let events = nodes[0].node.get_and_clear_pending_events();
 +      assert_eq!(events.len(), 1);
 +      match events[0] {
 +              Event::PaymentFailed { payment_hash: ev_payment_hash, payment_id: ev_payment_id } => {
 +                      assert_eq!(*payment_hash, ev_payment_hash);
 +                      assert_eq!(payment_id, ev_payment_id);
 +              }
 +              _ => panic!("Unexpected second event"),
 +      }
  }
  
  impl msgs::ChannelUpdate {
@@@ -285,7 -274,7 +285,7 @@@ fn test_fee_failures() 
  
        // positive case
        let (route, payment_hash_success, payment_preimage_success, payment_secret_success) = get_route_and_payment_hash!(nodes[0], nodes[2], 40_000);
 -      nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
        }
  
        let (payment_preimage_success, payment_hash_success, payment_secret_success) = get_payment_preimage_hash!(nodes[2]);
 -      nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash_success, &Some(payment_secret_success), PaymentId(payment_hash_success.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        pass_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], 40_000, payment_hash_success, payment_secret_success);
        claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_success);
@@@ -355,7 -344,7 +355,7 @@@ fn test_onion_failure() 
                // break the first (non-final) hop payload by swapping the realm (0) byte for a byte
                // describing a length-1 TLV payload, which is obviously bogus.
                new_payloads[0].data[0] = 1;
-               msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
+               msg.onion_routing_packet = onion_utils::construct_onion_packet_with_writable_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
        }, ||{}, true, Some(PERM|22), Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent: true}), Some(short_channel_id));
  
        // final node failure
                // break the last-hop payload by swapping the realm (0) byte for a byte describing a
                // length-1 TLV payload, which is obviously bogus.
                new_payloads[1].data[0] = 1;
-               msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
+               msg.onion_routing_packet = onion_utils::construct_onion_packet_with_writable_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
        }, ||{}, false, Some(PERM|22), Some(NetworkUpdate::ChannelFailure{short_channel_id, is_permanent: true}), Some(short_channel_id));
  
        // the following three with run_onion_failure_test_with_fail_intercept() test only the origin node
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
 -                                      &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
 +                                      &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
                                                forward_info.outgoing_cltv_value += 1,
                                        _ => {},
                                }
                for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                        for f in pending_forwards.iter_mut() {
                                match f {
 -                                      &mut HTLCForwardInfo::AddHTLC { ref mut forward_info, .. } =>
 -                                              forward_info.amt_to_forward -= 1,
 +                                      &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo { ref mut forward_info, .. }) =>
 +                                              forward_info.outgoing_amt_msat -= 1,
                                        _ => {},
                                }
                        }
@@@ -777,7 -766,7 +777,7 @@@ fn do_test_onion_failure_stale_channel_
                channel_monitors.insert(chanmon_1.get_funding_txo().0, &mut chanmon_1);
                channel_monitors.insert(chanmon_2.get_funding_txo().0, &mut chanmon_2);
  
 -              let chanmgr = <(_, ChannelManager<_, _, _, _, _, _>)>::read(
 +              let chanmgr = <(_, ChannelManager<_, _, _, _, _>)>::read(
                        &mut &nodes[1].node.encode()[..], ChannelManagerReadArgs {
                                default_config: *nodes[1].node.get_current_default_configuration(),
                                keys_manager: nodes[1].keys_manager,
@@@ -802,166 -791,52 +802,52 @@@ fn test_onion_failure_stale_channel_upd
  }
  
  #[test]
- fn test_default_to_onion_payload_tlv_format() {
-       // Tests that we default to creating tlv format onion payloads when no `NodeAnnouncementInfo`
-       // `features` for a node in the `network_graph` exists, or when the node isn't in the
-       // `network_graph`, and no other known `features` for the node exists.
-       let mut priv_channels_conf = UserConfig::default();
-       priv_channels_conf.channel_handshake_config.announced_channel = false;
-       let chanmon_cfgs = create_chanmon_cfgs(5);
-       let node_cfgs = create_node_cfgs(5, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(5, &node_cfgs, &[None, None, None, None, Some(priv_channels_conf)]);
-       let mut nodes = create_network(5, &node_cfgs, &node_chanmgrs);
-       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 2, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_unannounced_chan_between_nodes_with_value(&nodes, 3, 4, 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id());
-       let origin_node = &nodes[0];
-       let network_graph = origin_node.network_graph;
-       // Clears all the `NodeAnnouncementInfo` for all nodes of `nodes[0]`'s `network_graph`, so that
-       // their `features` aren't used when creating the `route`.
-       network_graph.clear_nodes_announcement_info();
-       let (announced_route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[3], payment_params, 10_000, TEST_FINAL_CLTV);
-       let hops = &announced_route.paths[0];
-       // Assert that the hop between `nodes[1]` and `nodes[2]` defaults to supporting variable length
-       // onions, as `nodes[0]` has no `NodeAnnouncementInfo` `features` for `node[2]`
-       assert!(hops[1].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[2]` and `nodes[3]` defaults to supporting variable length
-       // onions, as `nodes[0]` has no `NodeAnnouncementInfo` `features` for `node[3]`, and no `InvoiceFeatures`
-       // for the `payment_params`, which would otherwise have been used.
-       assert!(hops[2].node_features.supports_variable_length_onion());
-       // Note that we do not assert that `hops[0]` (the channel between `nodes[0]` and `nodes[1]`)
-       // supports variable length onions, as the `InitFeatures` exchanged in the init message
-       // between the nodes will be used when creating the route. We therefore do not default to
-       // supporting variable length onions for that hop, as the `InitFeatures` in this case are
-       // `channelmanager::provided_init_features()`.
-       let unannounced_chan = &nodes[4].node.list_usable_channels()[0];
-       let last_hop = RouteHint(vec![RouteHintHop {
-               src_node_id: nodes[3].node.get_our_node_id(),
-               short_channel_id: unannounced_chan.short_channel_id.unwrap(),
-               fees: RoutingFees {
-                       base_msat: 0,
-                       proportional_millionths: 0,
-               },
-               cltv_expiry_delta: 42,
-               htlc_minimum_msat: None,
-               htlc_maximum_msat: None,
-       }]);
-       let unannounced_chan_params = PaymentParameters::from_node_id(nodes[4].node.get_our_node_id()).with_route_hints(vec![last_hop]);
-       let (unannounced_route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[4], unannounced_chan_params, 10_000, TEST_FINAL_CLTV);
-       let unannounced_chan_hop = &unannounced_route.paths[0][3];
-       // Ensure that `nodes[4]` doesn't exist in `nodes[0]`'s `network_graph`, as it's not public.
-       assert!(&network_graph.read_only().nodes().get(&NodeId::from_pubkey(&nodes[4].node.get_our_node_id())).is_none());
-       // Assert that the hop between `nodes[3]` and `nodes[4]` defaults to supporting variable length
-       // onions, even though `nodes[4]` as `nodes[0]` doesn't exists in `nodes[0]`'s `network_graph`,
-       // and no `InvoiceFeatures` for the `payment_params` exists, which would otherwise have been
-       // used.
-       assert!(unannounced_chan_hop.node_features.supports_variable_length_onion());
-       let cur_height = nodes[0].best_block_info().1 + 1;
-       let (announced_route_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&announced_route.paths[0], 40000, &None, cur_height, &None).unwrap();
-       let (unannounced_route_paylods, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&unannounced_route.paths[0], 40000, &None, cur_height, &None).unwrap();
-       for onion_payloads in vec![announced_route_payloads, unannounced_route_paylods] {
-               for onion_payload in onion_payloads.iter() {
-                       match onion_payload.format {
-                               msgs::OnionHopDataFormat::Legacy {..} => {
-                                       panic!("Generated a `msgs::OnionHopDataFormat::Legacy` payload, even though that shouldn't have happend.");
-                               }
-                               _ => {}
-                       }
-               }
-       }
- }
+ fn test_always_create_tlv_format_onion_payloads() {
+       // Verify that we always generate tlv onion format payloads, even if the features specifically
+       // specifies no support for variable length onions, as the legacy payload format has been
+       // deprecated in BOLT4.
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let mut node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
  
- #[test]
- fn test_do_not_default_to_onion_payload_tlv_format_when_unsupported() {
-       // Tests that we do not default to creating tlv onions if either of these types features
-       // exists, which specifies no support for variable length onions for a specific hop, when
-       // creating a route:
-       // 1. `InitFeatures` to the counterparty node exchanged with the init message to the node.
-       // 2. `NodeFeatures` in the `NodeAnnouncementInfo` of a node in sender node's `network_graph`.
-       // 3. `InvoiceFeatures` specified by the receiving node, when no `NodeAnnouncementInfo`
-       // `features` exists for the receiver in the sender's `network_graph`.
-       let chanmon_cfgs = create_chanmon_cfgs(4);
-       let mut node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
-       // Set `node[1]` config to `InitFeatures::empty()` + `static_remote_key` which implies
-       // `!supports_variable_length_onion()` but still supports the required static-remote-key
-       // feature.
+       // Set `node[1]`'s config features to features which return `false` for
+       // `supports_variable_length_onion()`
+       let mut no_variable_length_onion_features = InitFeatures::empty();
+       no_variable_length_onion_features.set_static_remote_key_required();
        let mut node_1_cfg = &mut node_cfgs[1];
-       node_1_cfg.features = InitFeatures::empty();
-       node_1_cfg.features.set_static_remote_key_required();
+       node_1_cfg.features = no_variable_length_onion_features;
  
-       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
-       let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+       let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
  
-       create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features());
-       create_announced_chan_between_nodes(&nodes, 2, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
+       create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::empty(), InitFeatures::empty());
+       create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::empty(), InitFeatures::empty());
  
-       let payment_params = PaymentParameters::from_node_id(nodes[3].node.get_our_node_id())
+       let payment_params = PaymentParameters::from_node_id(nodes[2].node.get_our_node_id())
                .with_features(InvoiceFeatures::empty());
-       let origin_node = &nodes[0];
-       let network_graph = origin_node.network_graph;
-       network_graph.clear_nodes_announcement_info();
-       // Set `NodeAnnouncementInfo` `features` which do not support variable length onions for
-       // `nodes[2]` in `nodes[0]`'s `network_graph`.
-       let nodes_2_unsigned_node_announcement = msgs::UnsignedNodeAnnouncement {
-               features: NodeFeatures::empty(),
-               timestamp: 0,
-               node_id: nodes[2].node.get_our_node_id(),
-               rgb: [32; 3],
-               alias: [16;32],
-               addresses: Vec::new(),
-               excess_address_data: Vec::new(),
-               excess_data: Vec::new(),
-       };
-       let _res = network_graph.update_node_from_unsigned_announcement(&nodes_2_unsigned_node_announcement);
-       let (route, _, _, _) = get_route_and_payment_hash!(
-               origin_node, nodes[3], payment_params, 10_000, TEST_FINAL_CLTV);
+       let (route, _payment_hash, _payment_preimage, _payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], payment_params, 40000, TEST_FINAL_CLTV);
  
        let hops = &route.paths[0];
-       // Assert that the hop between `nodes[0]` and `nodes[1]` doesn't support variable length
-       // onions, as as the `InitFeatures` exchanged (`InitFeatures::empty()`) in the init message
-       // between the nodes when setting up the channel is used when creating the `route` and that we
-       // therefore do not default to supporting variable length onions. Despite `nodes[0]` having no
-       // `NodeAnnouncementInfo` `features` for `node[1]`.
+       // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[0].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[1]` and `nodes[2]` uses the `features` from
-       // `nodes_2_unsigned_node_announcement` that doesn't support variable length onions.
+       // Asserts that the first hop to `node[1]` signals no support for variable length onions.
        assert!(!hops[1].node_features.supports_variable_length_onion());
-       // Assert that the hop between `nodes[2]` and `nodes[3]` uses the `InvoiceFeatures` set to the
-       // `payment_params`, that doesn't support variable length onions. We therefore do not end up
-       // defaulting to supporting variable length onions, despite `nodes[0]` having no
-       // `NodeAnnouncementInfo` `features` for `node[3]`.
-       assert!(!hops[2].node_features.supports_variable_length_onion());
  
        let cur_height = nodes[0].best_block_info().1 + 1;
        let (onion_payloads, _htlc_msat, _htlc_cltv) = onion_utils::build_onion_payloads(&route.paths[0], 40000, &None, cur_height, &None).unwrap();
  
-       for onion_payload in onion_payloads.iter() {
-               match onion_payload.format {
-                       msgs::OnionHopDataFormat::Legacy {..} => {}
-                       _ => {
-                               panic!("Should have only have generated `msgs::OnionHopDataFormat::Legacy` payloads");
-                       }
-               }
+       match onion_payloads[0].format {
+               msgs::OnionHopDataFormat::NonFinalNode {..} => {},
+               _ => { panic!(
+                       "Should have generated a `msgs::OnionHopDataFormat::NonFinalNode` payload for `hops[0]`,
+                       despite that the features signals no support for variable length onions"
+               )}
+       }
+       match onion_payloads[1].format {
+               msgs::OnionHopDataFormat::FinalNode {..} => {},
+               _ => {panic!(
+                       "Should have generated a `msgs::OnionHopDataFormat::FinalNode` payload for `hops[1]`,
+                       despite that the features signals no support for variable length onions"
+               )}
        }
  }
  
@@@ -1022,7 -897,7 +908,7 @@@ fn test_phantom_onion_hmac_failure() 
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
                let mut forward_htlcs = nodes[1].node.forward_htlcs.lock().unwrap();
                let mut pending_forward = forward_htlcs.get_mut(&phantom_scid).unwrap();
                match pending_forward[0] {
 -                      HTLCForwardInfo::AddHTLC {
 +                      HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                forward_info: PendingHTLCInfo {
                                        routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                        ..
                                }, ..
 -                      } => {
 +                      }) => {
                                onion_packet.hmac[onion_packet.hmac.len() - 1] ^= 1;
                                Sha256::hash(&onion_packet.hop_data).into_inner().to_vec()
                        },
@@@ -1083,7 -958,7 +969,7 @@@ fn test_phantom_invalid_onion_payload(
        // We'll use the session priv later when constructing an invalid onion packet.
        let session_priv = [3; 32];
        *nodes[0].keys_manager.override_random_bytes.lock().unwrap() = Some(session_priv);
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
 -                              &mut HTLCForwardInfo::AddHTLC {
 +                              &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo {
                                                routing: PendingHTLCRouting::Forward { ref mut onion_packet, .. },
                                                ..
                                        }, ..
 -                              } => {
 +                              }) => {
                                        // Construct the onion payloads for the entire route and an invalid amount.
                                        let height = nodes[0].best_block_info().1;
                                        let session_priv = SecretKey::from_slice(&session_priv).unwrap();
@@@ -1154,7 -1029,7 +1040,7 @@@ fn test_phantom_final_incorrect_cltv_ex
        let (route, phantom_scid) = get_phantom_route!(nodes, recv_value_msat, channel);
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
        for (_, pending_forwards) in nodes[1].node.forward_htlcs.lock().unwrap().iter_mut() {
                for f in pending_forwards.iter_mut() {
                        match f {
 -                              &mut HTLCForwardInfo::AddHTLC {
 +                              &mut HTLCForwardInfo::AddHTLC(PendingAddHTLCInfo {
                                        forward_info: PendingHTLCInfo { ref mut outgoing_cltv_value, .. }, ..
 -                              } => {
 +                              }) => {
                                        *outgoing_cltv_value += 1;
                                },
                                _ => panic!("Unexpected forward"),
@@@ -1213,7 -1088,7 +1099,7 @@@ fn test_phantom_failure_too_low_cltv() 
        route.paths[0][1].cltv_expiry_delta = 5;
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@@ -1256,7 -1131,7 +1142,7 @@@ fn test_phantom_failure_too_low_recv_am
        let (mut route, phantom_scid) = get_phantom_route!(nodes, bad_recv_amt_msat, channel);
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@@ -1308,7 -1183,7 +1194,7 @@@ fn test_phantom_dust_exposure_failure(
        let (mut route, _) = get_phantom_route!(nodes, max_dust_exposure + 1, channel);
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
@@@ -1351,7 -1226,7 +1237,7 @@@ fn test_phantom_failure_reject_payment(
        let (mut route, phantom_scid) = get_phantom_route!(nodes, recv_amt_msat, channel);
  
        // Route the HTLC through to the destination.
 -      nodes[0].node.send_payment(&route, payment_hash.clone(), &Some(payment_secret)).unwrap();
 +      nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let update_0 = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
        let mut update_add = update_0.update_add_htlcs[0].clone();
index 953313955a50220ed9b7d9693897facc35fbf7e6,491188fcae13daf4cd17af099e129399197f5d47..23dc556cfacca4f129d7968323f997a0f3c63692
@@@ -82,6 -82,15 +82,15 @@@ pub(super) fn gen_ammag_from_shared_sec
        Hmac::from_engine(hmac).into_inner()
  }
  
+ #[cfg(test)]
+ #[inline]
+ pub(super) fn gen_pad_from_shared_secret(shared_secret: &[u8]) -> [u8; 32] {
+       assert_eq!(shared_secret.len(), 32);
+       let mut hmac = HmacEngine::<Sha256>::new(&[0x70, 0x61, 0x64]); // pad
+       hmac.input(&shared_secret);
+       Hmac::from_engine(hmac).into_inner()
+ }
  pub(crate) fn next_hop_packet_pubkey<T: secp256k1::Signing + secp256k1::Verification>(secp_ctx: &Secp256k1<T>, packet_pubkey: PublicKey, packet_shared_secret: &[u8; 32]) -> Result<PublicKey, secp256k1::Error> {
        let blinding_factor = {
                let mut sha = Sha256::engine();
@@@ -153,24 -162,18 +162,18 @@@ pub(super) fn build_onion_payloads(path
                let value_msat = if cur_value_msat == 0 { hop.fee_msat } else { cur_value_msat };
                let cltv = if cur_cltv == starting_htlc_offset { hop.cltv_expiry_delta + starting_htlc_offset } else { cur_cltv };
                res.insert(0, msgs::OnionHopData {
-                       format: if hop.node_features.supports_variable_length_onion() {
-                               if idx == 0 {
-                                       msgs::OnionHopDataFormat::FinalNode {
-                                               payment_data: if let &Some(ref payment_secret) = payment_secret_option {
-                                                       Some(msgs::FinalOnionHopData {
-                                                               payment_secret: payment_secret.clone(),
-                                                               total_msat,
-                                                       })
-                                               } else { None },
-                                               keysend_preimage: *keysend_preimage,
-                                       }
-                               } else {
-                                       msgs::OnionHopDataFormat::NonFinalNode {
-                                               short_channel_id: last_short_channel_id,
-                                       }
+                       format: if idx == 0 {
+                               msgs::OnionHopDataFormat::FinalNode {
+                                       payment_data: if let &Some(ref payment_secret) = payment_secret_option {
+                                               Some(msgs::FinalOnionHopData {
+                                                       payment_secret: payment_secret.clone(),
+                                                       total_msat,
+                                               })
+                                       } else { None },
+                                       keysend_preimage: *keysend_preimage,
                                }
                        } else {
-                               msgs::OnionHopDataFormat::Legacy {
+                               msgs::OnionHopDataFormat::NonFinalNode {
                                        short_channel_id: last_short_channel_id,
                                }
                        },
@@@ -230,9 -233,9 +233,9 @@@ pub(super) fn construct_onion_packet(pa
  }
  
  #[cfg(test)]
- // Used in testing to write bogus OnionHopDatas, which is otherwise not representable in
- // msgs::OnionHopData.
- pub(super) fn construct_onion_packet_bogus_hopdata<HD: Writeable>(payloads: Vec<HD>, onion_keys: Vec<OnionKeys>, prng_seed: [u8; 32], associated_data: &PaymentHash) -> msgs::OnionPacket {
+ /// Used in testing to write bogus `BogusOnionHopData` as well as `RawOnionHopData`, which is
+ /// otherwise not representable in `msgs::OnionHopData`.
+ pub(super) fn construct_onion_packet_with_writable_hopdata<HD: Writeable>(payloads: Vec<HD>, onion_keys: Vec<OnionKeys>, prng_seed: [u8; 32], associated_data: &PaymentHash) -> msgs::OnionPacket {
        let mut packet_data = [0; ONION_DATA_LEN];
  
        let mut chacha = ChaCha20::new(&prng_seed, &[0; 8]);
@@@ -589,6 -592,31 +592,6 @@@ pub(super) fn process_onion_failure<T: 
        } else { unreachable!(); }
  }
  
 -/// An input used when decoding an onion packet.
 -pub(crate) trait DecodeInput {
 -      type Arg;
 -      /// If Some, this is the input when checking the hmac of the onion packet.
 -      fn payment_hash(&self) -> Option<&PaymentHash>;
 -      /// Read argument when decrypting our hop payload.
 -      fn read_arg(self) -> Self::Arg;
 -}
 -
 -impl DecodeInput for PaymentHash {
 -      type Arg = ();
 -      fn payment_hash(&self) -> Option<&PaymentHash> {
 -              Some(self)
 -      }
 -      fn read_arg(self) -> Self::Arg { () }
 -}
 -
 -impl DecodeInput for SharedSecret {
 -      type Arg = SharedSecret;
 -      fn payment_hash(&self) -> Option<&PaymentHash> {
 -              None
 -      }
 -      fn read_arg(self) -> Self::Arg { self }
 -}
 -
  /// Allows `decode_next_hop` to return the next hop packet bytes for either payments or onion
  /// message forwards.
  pub(crate) trait NextPacketBytes: AsMut<[u8]> {
@@@ -639,7 -667,7 +642,7 @@@ pub(crate) enum OnionDecodeErr 
  }
  
  pub(crate) fn decode_next_payment_hop(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: PaymentHash) -> Result<Hop, OnionDecodeErr> {
 -      match decode_next_hop(shared_secret, hop_data, hmac_bytes, payment_hash) {
 +      match decode_next_hop(shared_secret, hop_data, hmac_bytes, Some(payment_hash), ()) {
                Ok((next_hop_data, None)) => Ok(Hop::Receive(next_hop_data)),
                Ok((next_hop_data, Some((next_hop_hmac, FixedSizeOnionPacket(new_packet_bytes))))) => {
                        Ok(Hop::Forward {
        }
  }
  
 -pub(crate) fn decode_next_hop<D: DecodeInput, R: ReadableArgs<D::Arg>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], decode_input: D) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
 +pub(crate) fn decode_next_untagged_hop<T, R: ReadableArgs<T>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], read_args: T) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
 +      decode_next_hop(shared_secret, hop_data, hmac_bytes, None, read_args)
 +}
 +
 +fn decode_next_hop<T, R: ReadableArgs<T>, N: NextPacketBytes>(shared_secret: [u8; 32], hop_data: &[u8], hmac_bytes: [u8; 32], payment_hash: Option<PaymentHash>, read_args: T) -> Result<(R, Option<([u8; 32], N)>), OnionDecodeErr> {
        let (rho, mu) = gen_rho_mu_from_shared_secret(&shared_secret);
        let mut hmac = HmacEngine::<Sha256>::new(&mu);
        hmac.input(hop_data);
 -      if let Some(payment_hash) = decode_input.payment_hash() {
 -              hmac.input(&payment_hash.0[..]);
 +      if let Some(tag) = payment_hash {
 +              hmac.input(&tag.0[..]);
        }
        if !fixed_time_eq(&Hmac::from_engine(hmac).into_inner(), &hmac_bytes) {
                return Err(OnionDecodeErr::Malformed {
  
        let mut chacha = ChaCha20::new(&rho, &[0u8; 8]);
        let mut chacha_stream = ChaChaReader { chacha: &mut chacha, read: Cursor::new(&hop_data[..]) };
 -      match R::read(&mut chacha_stream, decode_input.read_arg()) {
 +      match R::read(&mut chacha_stream, read_args) {
                Err(err) => {
                        let error_code = match err {
                                msgs::DecodeError::UnknownVersion => 0x4000 | 1, // unknown realm byte
@@@ -741,7 -765,7 +744,7 @@@ mod tests 
        use crate::ln::features::{ChannelFeatures, NodeFeatures};
        use crate::routing::router::{Route, RouteHop};
        use crate::ln::msgs;
-       use crate::util::ser::{Writeable, Writer};
+       use crate::util::ser::{Writeable, Writer, VecWriter};
  
        use hex;
  
  
        use super::OnionKeys;
  
+       fn get_test_session_key() -> SecretKey {
+               SecretKey::from_slice(&hex::decode("4141414141414141414141414141414141414141414141414141414141414141").unwrap()[..]).unwrap()
+       }
        fn build_test_onion_keys() -> Vec<OnionKeys> {
                // Keys from BOLT 4, used in both test vector tests
                let secp_ctx = Secp256k1::new();
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("032c0b7cf95324a07d05398b240174dc0c2be444d96b159aa6c7f7b1e668680991").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                                        RouteHop {
                                                pubkey: PublicKey::from_slice(&hex::decode("02edabbd16b41c8371b92ef2f04c1185b4f03b6dcd52ba9b78d9d7c89c8f221145").unwrap()[..]).unwrap(),
                                                channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
-                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // Test vectors are garbage and not generateble from a RouteHop, we fill in payloads manually
+                                               short_channel_id: 0, fee_msat: 0, cltv_expiry_delta: 0 // We fill in the payloads manually instead of generating them from RouteHops.
                                        },
                        ]],
                        payment_params: None,
                };
  
-               let session_priv = SecretKey::from_slice(&hex::decode("4141414141414141414141414141414141414141414141414141414141414141").unwrap()[..]).unwrap();
-               let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
+               let onion_keys = super::construct_onion_keys(&secp_ctx, &route.paths[0], &get_test_session_key()).unwrap();
                assert_eq!(onion_keys.len(), route.paths[0].len());
                onion_keys
        }
  
        #[test]
        fn onion_vectors() {
-               // Legacy packet creation test vectors from BOLT 4
                let onion_keys = build_test_onion_keys();
  
+               // Test generation of ephemeral keys and secrets. These values used to be part of the BOLT4
+               // test vectors, but have since been removed. We keep them as they provide test coverage.
                assert_eq!(onion_keys[0].shared_secret.secret_bytes(), hex::decode("53eb63ea8a3fec3b3cd433b85cd62a4b145e1dda09391b348c4e1cd36a03ea66").unwrap()[..]);
                assert_eq!(onion_keys[0].blinding_factor[..], hex::decode("2ec2e5da605776054187180343287683aa6a51b4b1c04d6dd49c45d8cffb3c36").unwrap()[..]);
                assert_eq!(onion_keys[0].ephemeral_pubkey.serialize()[..], hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]);
                assert_eq!(onion_keys[4].rho, hex::decode("034e18b8cc718e8af6339106e706c52d8df89e2b1f7e9142d996acf88df8799b").unwrap()[..]);
                assert_eq!(onion_keys[4].mu, hex::decode("8e45e5c61c2b24cb6382444db6698727afb063adecd72aada233d4bf273d975a").unwrap()[..]);
  
-               // Test vectors below are flat-out wrong: they claim to set outgoing_cltv_value to non-0 :/
+               // Packet creation test vectors from BOLT 4 (see
+               // https://github.com/lightning/bolts/blob/16973e2b857e853308cafd59e42fa830d75b1642/bolt04/onion-test.json).
+               // Note that we represent the test vector payloads 2 and 5 through RawOnionHopData::data
+               // with raw hex instead of our in-memory enums, as the payloads contains custom types, and
+               // we have no way of representing that with our enums.
                let payloads = vec!(
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0,
-                               },
-                               amt_to_forward: 0,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0101010101010101,
-                               },
-                               amt_to_forward: 0x0100000001,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0202020202020202,
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 1,
                                },
-                               amt_to_forward: 0x0200000002,
-                               outgoing_cltv_value: 0,
+                               amt_to_forward: 15000,
+                               outgoing_cltv_value: 1500,
+                       }),
+                       /*
+                       The second payload is represented by raw hex as it contains custom type data. Content:
+                       1. length "52" (payload_length 82).
+                       The first part of the payload has the `NonFinalNode` format, with content as follows:
+                       2. amt_to_forward "020236b0"
+                          02 (type amt_to_forward) 02 (length 2) 36b0 (value 14000)
+                       3. outgoing_cltv_value "04020578"
+                          04 (type outgoing_cltv_value) 02 (length 2) 0578 (value 1400)
+                       4. short_channel_id "06080000000000000002"
+                          06 (type short_channel_id) 08 (length 8) 0000000000000002 (value 2)
+                       The rest of the payload is custom type data:
+                       5. custom_record "fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f"
+                       */
+                       RawOnionHopData {
+                               data: hex::decode("52020236b00402057806080000000000000002fd02013c0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f0102030405060708090a0b0c0d0e0f").unwrap(),
                        },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0303030303030303,
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 3,
                                },
-                               amt_to_forward: 0x0300000003,
-                               outgoing_cltv_value: 0,
-                       },
-                       msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0404040404040404,
+                               amt_to_forward: 12500,
+                               outgoing_cltv_value: 1250,
+                       }),
+                       RawOnionHopData::new(msgs::OnionHopData {
+                               format: msgs::OnionHopDataFormat::NonFinalNode {
+                                       short_channel_id: 4,
                                },
-                               amt_to_forward: 0x0400000004,
-                               outgoing_cltv_value: 0,
+                               amt_to_forward: 10000,
+                               outgoing_cltv_value: 1000,
+                       }),
+                       /*
+                       The fifth payload is represented by raw hex as it contains custom type data. Content:
+                       1. length "fd0110" (payload_length 272).
+                       The first part of the payload has the `FinalNode` format, with content as follows:
+                       1. amt_to_forward "02022710"
+                          02 (type amt_to_forward) 02 (length 2) 2710 (value 10000)
+                       2. outgoing_cltv_value "040203e8"
+                          04 (type outgoing_cltv_value) 02 (length 2) 03e8 (value 1000)
+                       3. payment_data "082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710"
+                          08 (type short_channel_id) 22 (length 34) 24a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f61704 (payment_secret) 2710 (total_msat value 10000)
+                       The rest of the payload is custom type data:
+                       4. custom_record "fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a"
+                       */
+                       RawOnionHopData {
+                               data: hex::decode("fd011002022710040203e8082224a33562c54507a9334e79f0dc4f17d407e6d7c61f0e2f3d0d38599502f617042710fd012de02a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a2a").unwrap(),
                        },
                );
  
-               let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32])));
-               // Just check the final packet encoding, as it includes all the per-hop vectors in it
-               // anyway...
-               assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a716a996c7845c93d90e4ecbb9bde4ece2f69425c99e4bc820e44485455f135edc0d10f7d61ab590531cf08000179a333a347f8b4072f216400406bdf3bf038659793d4a1fd7b246979e3150a0a4cb052c9ec69acf0f48c3d39cd55675fe717cb7d80ce721caad69320c3a469a202f1e468c67eaf7a7cd8226d0fd32f7b48084dca885d56047694762b67021713ca673929c163ec36e04e40ca8e1c6d17569419d3039d9a1ec866abe044a9ad635778b961fc0776dc832b3a451bd5d35072d2269cf9b040f6b7a7dad84fb114ed413b1426cb96ceaf83825665ed5a1d002c1687f92465b49ed4c7f0218ff8c6c7dd7221d589c65b3b9aaa71a41484b122846c7c7b57e02e679ea8469b70e14fe4f70fee4d87b910cf144be6fe48eef24da475c0b0bcc6565ae82cd3f4e3b24c76eaa5616c6111343306ab35c1fe5ca4a77c0e314ed7dba39d6f1e0de791719c241a939cc493bea2bae1c1e932679ea94d29084278513c77b899cc98059d06a27d171b0dbdf6bee13ddc4fc17a0c4d2827d488436b57baa167544138ca2e64a11b43ac8a06cd0c2fba2d4d900ed2d9205305e2d7383cc98dacb078133de5f6fb6bed2ef26ba92cea28aafc3b9948dd9ae5559e8bd6920b8cea462aa445ca6a95e0e7ba52961b181c79e73bd581821df2b10173727a810c92b83b5ba4a0403eb710d2ca10689a35bec6c3a708e9e92f7d78ff3c5d9989574b00c6736f84c199256e76e19e78f0c98a9d580b4a658c84fc8f2096c2fbea8f5f8c59d0fdacb3be2802ef802abbecb3aba4acaac69a0e965abd8981e9896b1f6ef9d60f7a164b371af869fd0e48073742825e9434fc54da837e120266d53302954843538ea7c6c3dbfb4ff3b2fdbe244437f2a153ccf7bdb4c92aa08102d4f3cff2ae5ef86fab4653595e6a5837fa2f3e29f27a9cde5966843fb847a4a61f1e76c281fe8bb2b0a181d096100db5a1a5ce7a910238251a43ca556712eaadea167fb4d7d75825e440f3ecd782036d7574df8bceacb397abefc5f5254d2722215c53ff54af8299aaaad642c6d72a14d27882d9bbd539e1cc7a527526ba89b8c037ad09120e98ab042d3e8652b31ae0e478516bfaf88efca9f3676ffe99d2819dcaeb7610a626695f53117665d267d3f7abebd6bbd6733f645c72c389f03855bdf1e4b8075b516569b118233a0f0971d24b83113c0b096f5216a207ca99a7cddc81c130923fe3d91e7508c9ac5f2e914ff5dccab9e558566fa14efb34ac98d878580814b94b73acbfde9072f30b881f7f0fff42d4045d1ace6322d86a97d164aa84d93a60498065cc7c20e636f5862dc81531a88c60305a2e59a985be327a6902e4bed986dbf4a0b50c217af0ea7fdf9ab37f9ea1a1aaa72f54cf40154ea9b269f1a7c09f9f43245109431a175d50e2db0132337baa0ef97eed0fcf20489da36b79a1172faccc2f7ded7c60e00694282d93359c4682135642bc81f433574aa8ef0c97b4ade7ca372c5ffc23c7eddd839bab4e0f14d6df15c9dbeab176bec8b5701cf054eb3072f6dadc98f88819042bf10c407516ee58bce33fbe3b3d86a54255e577db4598e30a135361528c101683a5fcde7e8ba53f3456254be8f45fe3a56120ae96ea3773631fcb3873aa3abd91bcff00bd38bd43697a2e789e00da6077482e7b1b1a677b5afae4c54e6cbdf7377b694eb7d7a5b913476a5be923322d3de06060fd5e819635232a2cf4f0731da13b8546d1d6d4f8d75b9fce6c2341a71b0ea6f780df54bfdb0dd5cd9855179f602f9172307c7268724c3618e6817abd793adc214a0dc0bc616816632f27ea336fb56dfd").unwrap());
+               // Verify that the serialized OnionHopDataFormat::NonFinalNode tlv payloads matches the test vectors
+               let mut w = VecWriter(Vec::new());
+               payloads[0].write(&mut w).unwrap();
+               let hop_1_serialized_payload = w.0;
+               let expected_serialized_hop_1_payload = &hex::decode("1202023a98040205dc06080000000000000001").unwrap()[..];
+               assert_eq!(hop_1_serialized_payload, expected_serialized_hop_1_payload);
+               w = VecWriter(Vec::new());
+               payloads[2].write(&mut w).unwrap();
+               let hop_3_serialized_payload = w.0;
+               let expected_serialized_hop_3_payload = &hex::decode("12020230d4040204e206080000000000000003").unwrap()[..];
+               assert_eq!(hop_3_serialized_payload, expected_serialized_hop_3_payload);
+               w = VecWriter(Vec::new());
+               payloads[3].write(&mut w).unwrap();
+               let hop_4_serialized_payload = w.0;
+               let expected_serialized_hop_4_payload = &hex::decode("1202022710040203e806080000000000000004").unwrap()[..];
+               assert_eq!(hop_4_serialized_payload, expected_serialized_hop_4_payload);
+               let pad_keytype_seed = super::gen_pad_from_shared_secret(&get_test_session_key().secret_bytes());
+               let packet: msgs::OnionPacket = super::construct_onion_packet_with_writable_hopdata::<_>(payloads, onion_keys, pad_keytype_seed, &PaymentHash([0x42; 32]));
+               assert_eq!(packet.encode(), hex::decode("0002EEC7245D6B7D2CCB30380BFBE2A3648CD7A942653F5AA340EDCEA1F283686619F7F3416A5AA36DC7EEB3EC6D421E9615471AB870A33AC07FA5D5A51DF0A8823AABE3FEA3F90D387529D4F72837F9E687230371CCD8D263072206DBED0234F6505E21E282ABD8C0E4F5B9FF8042800BBAB065036EADD0149B37F27DDE664725A49866E052E809D2B0198AB9610FAA656BBF4EC516763A59F8F42C171B179166BA38958D4F51B39B3E98706E2D14A2DAFD6A5DF808093ABFCA5AEAACA16EDED5DB7D21FB0294DD1A163EDF0FB445D5C8D7D688D6DD9C541762BF5A5123BF9939D957FE648416E88F1B0928BFA034982B22548E1A4D922690EECF546275AFB233ACF4323974680779F1A964CFE687456035CC0FBA8A5428430B390F0057B6D1FE9A8875BFA89693EEB838CE59F09D207A503EE6F6299C92D6361BC335FCBF9B5CD44747AADCE2CE6069CFDC3D671DAEF9F8AE590CF93D957C9E873E9A1BC62D9640DC8FC39C14902D49A1C80239B6C5B7FD91D05878CBF5FFC7DB2569F47C43D6C0D27C438ABFF276E87364DEB8858A37E5A62C446AF95D8B786EAF0B5FCF78D98B41496794F8DCAAC4EEF34B2ACFB94C7E8C32A9E9866A8FA0B6F2A06F00A1CCDE569F97EEC05C803BA7500ACC96691D8898D73D8E6A47B8F43C3D5DE74458D20EDA61474C426359677001FBD75A74D7D5DB6CB4FEB83122F133206203E4E2D293F838BF8C8B3A29ACB321315100B87E80E0EDB272EE80FDA944E3FB6084ED4D7F7C7D21C69D9DA43D31A90B70693F9B0CC3EAC74C11AB8FF655905688916CFA4EF0BD04135F2E50B7C689A21D04E8E981E74C6058188B9B1F9DFC3EEC6838E9FFBCF22CE738D8A177C19318DFFEF090CEE67E12DE1A3E2A39F61247547BA5257489CBC11D7D91ED34617FCC42F7A9DA2E3CF31A94A210A1018143173913C38F60E62B24BF0D7518F38B5BAB3E6A1F8AEB35E31D6442C8ABB5178EFC892D2E787D79C6AD9E2FC271792983FA9955AC4D1D84A36C024071BC6E431B625519D556AF38185601F70E29035EA6A09C8B676C9D88CF7E05E0F17098B584C4168735940263F940033A220F40BE4C85344128B14BEB9E75696DB37014107801A59B13E89CD9D2258C169D523BE6D31552C44C82FF4BB18EC9F099F3BF0E5B1BB2BA9A87D7E26F98D294927B600B5529C47E04D98956677CBCEE8FA2B60F49776D8B8C367465B7C626DA53700684FB6C918EAD0EAB8360E4F60EDD25B4F43816A75ECF70F909301825B512469F8389D79402311D8AECB7B3EF8599E79485A4388D87744D899F7C47EE644361E17040A7958C8911BE6F463AB6A9B2AFACD688EC55EF517B38F1339EFC54487232798BB25522FF4572FF68567FE830F92F7B8113EFCE3E98C3FFFBAEDCE4FD8B50E41DA97C0C08E423A72689CC68E68F752A5E3A9003E64E35C957CA2E1C48BB6F64B05F56B70B575AD2F278D57850A7AD568C24A4D32A3D74B29F03DC125488BC7C637DA582357F40B0A52D16B3B40BB2C2315D03360BC24209E20972C200566BCF3BBE5C5B0AEDD83132A8A4D5B4242BA370B6D67D9B67EB01052D132C7866B9CB502E44796D9D356E4E3CB47CC527322CD24976FE7C9257A2864151A38E568EF7A79F10D6EF27CC04CE382347A2488B1F404FDBF407FE1CA1C9D0D5649E34800E25E18951C98CAE9F43555EEF65FEE1EA8F15828807366C3B612CD5753BF9FB8FCED08855F742CDDD6F765F74254F03186683D646E6F09AC2805586C7CF11998357CAFC5DF3F285329366F475130C928B2DCEBA4AA383758E7A9D20705C4BB9DB619E2992F608A1BA65DB254BB389468741D0502E2588AEB54390AC600C19AF5C8E61383FC1BEBE0029E4474051E4EF908828DB9CCA13277EF65DB3FD47CCC2179126AAEFB627719F421E20").unwrap());
        }
  
        #[test]
                        writer.write_all(&self.data[..])
                }
        }
-       #[test]
-       fn variable_length_onion_vectors() {
-               // Packet creation test vectors from BOLT 4 (as of this writing at
-               // bolt04/onion-test-multi-frame.json in the spec repo).
-               // Note that we use he RawOnionHopData for everything except Legacy hops, as even the hops
-               // with "type": "tlv" are not valid TLV (they were for a previous version of TLV that
-               // didn't move forward), and, thus, cannot be directly represented in our in-memory enums.
-               let onion_keys = build_test_onion_keys();
-               let payloads = vec!(
-                       RawOnionHopData::new(msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0,
-                               },
-                               amt_to_forward: 0,
-                               outgoing_cltv_value: 0,
-                       }),
-                       RawOnionHopData {
-                               data: hex::decode("140101010101010101000000000000000100000001").unwrap(),
-                       },
-                       RawOnionHopData {
-                               data: hex::decode("fd0100000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff").unwrap(),
-                       },
-                       RawOnionHopData {
-                               data: hex::decode("140303030303030303000000000000000300000003").unwrap(),
-                       },
-                       RawOnionHopData::new(msgs::OnionHopData {
-                               format: msgs::OnionHopDataFormat::Legacy {
-                                       short_channel_id: 0x0404040404040404,
-                               },
-                               amt_to_forward: 4,
-                               outgoing_cltv_value: 4,
-                       }),
-               );
-               let packet: msgs::OnionPacket = super::construct_onion_packet_with_init_noise::<_, _>(payloads, onion_keys, super::FixedSizeOnionPacket([0; super::ONION_DATA_LEN]), Some(&PaymentHash([0x42; 32])));
-               // Just check the final packet encoding, as it includes all the per-hop vectors in it
-               // anyway...
-               assert_eq!(packet.encode(), hex::decode("0002eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619e5f14350c2a76fc232b5e46d421e9615471ab9e0bc887beff8c95fdb878f7b3a71a060daf367132b378b3a3883c0e2c0e026b8900b2b5cdbc784e1a3bb913f88a9c50f7d61ab590531cf08000178a333a347f8b4072ed056f820f77774345e183a342ec4729f3d84accf515e88adddb85ecc08daba68404bae9a8e8d7178977d7094a1ae549f89338c0777551f874159eb42d3a59fb9285ad4e24883f27de23942ec966611e99bee1cee503455be9e8e642cef6cef7b9864130f692283f8a973d47a8f1c1726b6e59969385975c766e35737c8d76388b64f748ee7943ffb0e2ee45c57a1abc40762ae598723d21bd184e2b338f68ebff47219357bd19cd7e01e2337b806ef4d717888e129e59cd3dc31e6201ccb2fd6d7499836f37a993262468bcb3a4dcd03a22818aca49c6b7b9b8e9e870045631d8e039b066ff86e0d1b7291f71cefa7264c70404a8e538b566c17ccc5feab231401e6c08a01bd5edfc1aa8e3e533b96e82d1f91118d508924b923531929aea889fcdf050597c681185f336b1da63b0939aa2b7c50b21b5eb7b6ad66c81fab98a3cdf73f658149e7e9ced4edde5d38c9b8f92e16f6b4ab13d7fca6a0e4ecc9f9de611a90da6e99c39551094c56e3196f282c5dffd9fc4b2fc12f3bca8e6fe47eb45fbdd3be21a8a8d200797eae3c9a0497132f92410d804977408494dff49dd3d8bce248e0b74fd9e6f0f7102c25ddfa02bd9ad9f746abbfa337ef811d5345a9e16b60de1767b209645ba40bd1f9a5f75bc04feca9b27c5554be4fe83fac2cb83aa447a817bb85ae966c68b420063833fada375e2f515965e687a45699632902672c654d1d18d7bcbf55e8fa57f63f2da449f8e1e606e8722df081e5f193fc4179feb99ad22819afdeef211f7c54afdba92aeef0c00b7bc2b65a4813c01f907a8377585708f2d4c940a25328e585714c8ded0a9a4d7a6de1027c1cb7a0198cd3db68b58c0704dfd0cfbe624e9cd18cc0ae5d96697bb476708b9ee0403d211e64e0d5a7683a7a9a140c02f0ff1c6e67a302941b4052bdea8a63e70a3ad62c5b89c698f1fd3c7685cb49705096cad702d02d93bcb1c27a409f4c9bddec001205ca4a2740f19b50900be81c7e847f1a863deea8d35701f1355cad8db57b1d4eb2ab4e29587734785abfb46ddede71928213d7d089dfdeda052827f459f1688cc0935bd47e7bcec27427c8376dcce7e22699567c0d145f8a7db33f6758815f1f15f9f7a9760dec4f34ae095edda4c64e9735bdd029c4e32c2ee31ba47ec5e6bdb97813d52dbd15b4e0b7a2c7f790ae64104d99f38c127f0a093288fa34144adb16b8968d4fa7656fcec99de8503dd46d3b03620a71c7cd085364abd30dccf7fbda25a1cdc102600149c9af1c97aa0372cd2e1909f28ac5c686f432b310e79528c9b8b9e8f314c1e74621ce6308ad2278b81d460892e0d9dd38b7c76d58be6dfd10ae7583ee1e7ef5b3f6f78dc60af0950df1b00cc55b6d178ba2e476bea0eaeef49323b83f05804159e7aef4eed4cc60dd07be76f067dfd0bcfb0b806b69ba921336a20c43c832d0cab8fa3ddeb29e3bf07b0d98a112eb07802756235a49d44a8b82a950d84e95e01971f0e106ccb337f07384e21620e0ad39e16ed9edca123226cf55ac44f449eeb53e38a7f27d101806e4823e4efcc887414240ee6826c4a5cb1c6443ad36ebf905a435c1d9054e54173911b17b5b40f60b3d9fd5f12eac54ca1e20191f5f18544d5fd3d665e9bcef96fb44b76110aa64d9db4c86c9513cbdad546538e8aec521fbe83ceac5e74a15629f1ed0b870a1d0d1e5680b6d6100d1bd3f3b9043bd35b8919c4088f1949b8be89e4701eb870f8ed64fafa446c78df3ea").unwrap());
-       }
  }
index 9d3324c7b6454947e84e2981839663a625ede569,fc891849cbb73758440b258a06692b3c6ce361d1..cffd9ddb0a581ea2e26a53ee3f305a8c7f81bbf7
@@@ -17,7 -17,7 +17,7 @@@ use bitcoin::secp256k1::PublicKey
  use crate::ln::channelmanager::ChannelDetails;
  use crate::ln::features::{ChannelFeatures, InvoiceFeatures, NodeFeatures};
  use crate::ln::msgs::{DecodeError, ErrorAction, LightningError, MAX_VALUE_MSAT};
 -use crate::routing::gossip::{DirectedChannelInfoWithUpdate, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
 +use crate::routing::gossip::{DirectedChannelInfo, EffectiveCapacity, ReadOnlyNetworkGraph, NetworkGraph, NodeId, RoutingFees};
  use crate::routing::scoring::{ChannelUsage, Score};
  use crate::util::ser::{Writeable, Readable, Writer};
  use crate::util::logger::{Level, Logger};
@@@ -29,49 -29,6 +29,49 @@@ use alloc::collections::BinaryHeap
  use core::cmp;
  use core::ops::Deref;
  
 +/// A trait defining behavior for routing a payment.
 +pub trait Router {
 +      /// Finds a [`Route`] between `payer` and `payee` for a payment with the given values.
 +      fn find_route(
 +              &self, payer: &PublicKey, route_params: &RouteParameters,
 +              first_hops: Option<&[&ChannelDetails]>, inflight_htlcs: InFlightHtlcs
 +      ) -> Result<Route, LightningError>;
 +}
 +
 +/// A map with liquidity value (in msat) keyed by a short channel id and the direction the HTLC
 +/// is traveling in. The direction boolean is determined by checking if the HTLC source's public
 +/// key is less than its destination. See [`InFlightHtlcs::used_liquidity_msat`] for more
 +/// details.
 +#[cfg(not(any(test, feature = "_test_utils")))]
 +pub struct InFlightHtlcs(HashMap<(u64, bool), u64>);
 +#[cfg(any(test, feature = "_test_utils"))]
 +pub struct InFlightHtlcs(pub HashMap<(u64, bool), u64>);
 +
 +impl InFlightHtlcs {
 +      /// Create a new `InFlightHtlcs` via a mapping from:
 +      /// (short_channel_id, source_pubkey < target_pubkey) -> used_liquidity_msat
 +      pub fn new(inflight_map: HashMap<(u64, bool), u64>) -> Self {
 +              InFlightHtlcs(inflight_map)
 +      }
 +
 +      /// Returns liquidity in msat given the public key of the HTLC source, target, and short channel
 +      /// id.
 +      pub fn used_liquidity_msat(&self, source: &NodeId, target: &NodeId, channel_scid: u64) -> Option<u64> {
 +              self.0.get(&(channel_scid, source < target)).map(|v| *v)
 +      }
 +}
 +
 +impl Writeable for InFlightHtlcs {
 +      fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> { self.0.write(writer) }
 +}
 +
 +impl Readable for InFlightHtlcs {
 +      fn read<R: io::Read>(reader: &mut R) -> Result<Self, DecodeError> {
 +              let infight_map: HashMap<(u64, bool), u64> = Readable::read(reader)?;
 +              Ok(Self(infight_map))
 +      }
 +}
 +
  /// A hop in a route
  #[derive(Clone, Debug, Hash, PartialEq, Eq)]
  pub struct RouteHop {
@@@ -228,8 -185,7 +228,7 @@@ pub const DEFAULT_MAX_PATH_COUNT: u8 = 
  const MEDIAN_HOP_CLTV_EXPIRY_DELTA: u32 = 40;
  
  // During routing, we only consider paths shorter than our maximum length estimate.
- // In the legacy onion format, the maximum number of hops used to be a fixed value of 20.
- // However, in the TLV onion format, there is no fixed maximum length, but the `hop_payloads`
+ // In the TLV onion format, there is no fixed maximum length, but the `hop_payloads`
  // field is always 1300 bytes. As the `tlv_payload` for each hop may vary in length, we have to
  // estimate how many hops the route may have so that it actually fits the `hop_payloads` field.
  //
@@@ -464,7 -420,7 +463,7 @@@ enum CandidateRouteHop<'a> 
        },
        /// A hop found in the [`ReadOnlyNetworkGraph`], where the channel capacity may be unknown.
        PublicHop {
 -              info: DirectedChannelInfoWithUpdate<'a>,
 +              info: DirectedChannelInfo<'a>,
                short_channel_id: u64,
        },
        /// A hop to the payee found in the payment invoice, though not necessarily a direct channel.
@@@ -537,8 -493,10 +536,8 @@@ fn max_htlc_from_capacity(capacity: Eff
                EffectiveCapacity::Unknown => EffectiveCapacity::Unknown.as_msat(),
                EffectiveCapacity::MaximumHTLC { amount_msat } =>
                        amount_msat.checked_shr(saturation_shift).unwrap_or(0),
 -              EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: None } =>
 -                      capacity_msat.checked_shr(saturation_shift).unwrap_or(0),
 -              EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat: Some(htlc_max) } =>
 -                      cmp::min(capacity_msat.checked_shr(saturation_shift).unwrap_or(0), htlc_max),
 +              EffectiveCapacity::Total { capacity_msat, htlc_maximum_msat } =>
 +                      cmp::min(capacity_msat.checked_shr(saturation_shift).unwrap_or(0), htlc_maximum_msat),
        }
  }
  
@@@ -1317,11 -1275,13 +1316,11 @@@ where L::Target: Logger 
                                        for chan_id in $node.channels.iter() {
                                                let chan = network_channels.get(chan_id).unwrap();
                                                if !chan.features.requires_unknown_bits() {
 -                                                      let (directed_channel, source) =
 -                                                              chan.as_directed_to(&$node_id).expect("inconsistent NetworkGraph");
 -                                                      if first_hops.is_none() || *source != our_node_id {
 -                                                              if let Some(direction) = directed_channel.direction() {
 -                                                                      if direction.enabled {
 +                                                      if let Some((directed_channel, source)) = chan.as_directed_to(&$node_id) {
 +                                                              if first_hops.is_none() || *source != our_node_id {
 +                                                                      if directed_channel.direction().enabled {
                                                                                let candidate = CandidateRouteHop::PublicHop {
 -                                                                                      info: directed_channel.with_update().unwrap(),
 +                                                                                      info: directed_channel,
                                                                                        short_channel_id: *chan_id,
                                                                                };
                                                                                add_entry!(candidate, *source, $node_id,
                                        let candidate = network_channels
                                                .get(&hop.short_channel_id)
                                                .and_then(|channel| channel.as_directed_to(&target))
 -                                              .and_then(|(channel, _)| channel.with_update())
 -                                              .map(|info| CandidateRouteHop::PublicHop {
 +                                              .map(|(info, _)| CandidateRouteHop::PublicHop {
                                                        info,
                                                        short_channel_id: hop.short_channel_id,
                                                })
@@@ -1854,8 -1815,10 +1853,8 @@@ fn add_random_cltv_offset(route: &mut R
                                                        random_channel.as_directed_from(&cur_node_id).map(|(dir_info, next_id)| {
                                                                if !nodes_to_avoid.iter().any(|x| x == next_id) {
                                                                        nodes_to_avoid[random_hop] = *next_id;
 -                                                                      dir_info.direction().map(|channel_update_info| {
 -                                                                              random_hop_offset = channel_update_info.cltv_expiry_delta.into();
 -                                                                              cur_hop = Some(*next_id);
 -                                                                      });
 +                                                                      random_hop_offset = dir_info.direction().cltv_expiry_delta.into();
 +                                                                      cur_hop = Some(*next_id);
                                                                }
                                                        });
                                                }
@@@ -5250,12 -5213,14 +5249,12 @@@ mod tests 
                                        for channel_id in &cur_node.channels {
                                                if let Some(channel_info) = network_channels.get(&channel_id) {
                                                        if let Some((dir_info, next_id)) = channel_info.as_directed_from(&cur_node_id) {
 -                                                              if let Some(channel_update_info) = dir_info.direction() {
 -                                                                      let next_cltv_expiry_delta = channel_update_info.cltv_expiry_delta as u32;
 -                                                                      if cur_path_cltv_deltas.iter().sum::<u32>()
 -                                                                              .saturating_add(next_cltv_expiry_delta) <= observed_cltv_expiry_delta {
 -                                                                              let mut new_path_cltv_deltas = cur_path_cltv_deltas.clone();
 -                                                                              new_path_cltv_deltas.push(next_cltv_expiry_delta);
 -                                                                              candidates.push_back((*next_id, new_path_cltv_deltas));
 -                                                                      }
 +                                                              let next_cltv_expiry_delta = dir_info.direction().cltv_expiry_delta as u32;
 +                                                              if cur_path_cltv_deltas.iter().sum::<u32>()
 +                                                                      .saturating_add(next_cltv_expiry_delta) <= observed_cltv_expiry_delta {
 +                                                                      let mut new_path_cltv_deltas = cur_path_cltv_deltas.clone();
 +                                                                      new_path_cltv_deltas.push(next_cltv_expiry_delta);
 +                                                                      candidates.push_back((*next_id, new_path_cltv_deltas));
                                                                }
                                                        }
                                                }
                let usage = ChannelUsage {
                        amount_msat: 0,
                        inflight_htlc_msat: 0,
 -                      effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: Some(1_000) },
 +                      effective_capacity: EffectiveCapacity::Total { capacity_msat: 1_024_000, htlc_maximum_msat: 1_000 },
                };
                scorer.set_manual_penalty(&NodeId::from_pubkey(&nodes[3]), 123);
                scorer.set_manual_penalty(&NodeId::from_pubkey(&nodes[4]), 456);