]> git.bitcoin.ninja Git - rust-lightning/commitdiff
Merge pull request #1080 from valentinewallace/2021-09-dup-chan-outpoint
authorMatt Corallo <649246+TheBlueMatt@users.noreply.github.com>
Thu, 30 Sep 2021 00:01:35 +0000 (00:01 +0000)
committerGitHub <noreply@github.com>
Thu, 30 Sep 2021 00:01:35 +0000 (00:01 +0000)
Fix fuzzer-found panic from duplicate channel outpoint

67 files changed:
CHANGELOG.md
fuzz/README.md
fuzz/src/chanmon_consistency.rs
fuzz/src/chanmon_deser.rs
fuzz/src/full_stack.rs
fuzz/src/msg_targets/gen_target.sh
fuzz/src/msg_targets/mod.rs
fuzz/src/msg_targets/msg_accept_channel.rs
fuzz/src/msg_targets/msg_announcement_signatures.rs
fuzz/src/msg_targets/msg_commitment_signed.rs
fuzz/src/msg_targets/msg_funding_created.rs
fuzz/src/msg_targets/msg_funding_locked.rs
fuzz/src/msg_targets/msg_funding_signed.rs
fuzz/src/msg_targets/msg_gossip_timestamp_filter.rs
fuzz/src/msg_targets/msg_open_channel.rs
fuzz/src/msg_targets/msg_query_channel_range.rs
fuzz/src/msg_targets/msg_reply_short_channel_ids_end.rs
fuzz/src/msg_targets/msg_revoke_and_ack.rs
fuzz/src/msg_targets/msg_shutdown.rs
fuzz/src/msg_targets/msg_update_add_htlc.rs
fuzz/src/msg_targets/msg_update_fail_htlc.rs
fuzz/src/msg_targets/msg_update_fail_malformed_htlc.rs
fuzz/src/msg_targets/msg_update_fee.rs
fuzz/src/msg_targets/msg_update_fulfill_htlc.rs
fuzz/src/msg_targets/utils.rs
fuzz/src/router.rs
lightning-background-processor/Cargo.toml
lightning-background-processor/src/lib.rs
lightning-block-sync/Cargo.toml
lightning-block-sync/src/convert.rs
lightning-block-sync/src/http.rs
lightning-invoice/Cargo.toml
lightning-invoice/src/utils.rs
lightning-net-tokio/Cargo.toml
lightning-net-tokio/src/lib.rs
lightning-persister/Cargo.toml
lightning-persister/src/lib.rs
lightning/Cargo.toml
lightning/src/chain/chainmonitor.rs
lightning/src/chain/channelmonitor.rs
lightning/src/chain/keysinterface.rs
lightning/src/chain/onchaintx.rs
lightning/src/chain/transaction.rs
lightning/src/ln/chan_utils.rs
lightning/src/ln/chanmon_update_fail_tests.rs
lightning/src/ln/channel.rs
lightning/src/ln/channelmanager.rs
lightning/src/ln/features.rs
lightning/src/ln/functional_test_utils.rs
lightning/src/ln/functional_tests.rs
lightning/src/ln/monitor_tests.rs
lightning/src/ln/msgs.rs
lightning/src/ln/onion_route_tests.rs
lightning/src/ln/onion_utils.rs
lightning/src/ln/peer_handler.rs
lightning/src/ln/reorg_tests.rs
lightning/src/ln/script.rs
lightning/src/ln/shutdown_tests.rs
lightning/src/ln/wire.rs
lightning/src/routing/network_graph.rs
lightning/src/routing/router.rs
lightning/src/util/enforcing_trait_impls.rs
lightning/src/util/events.rs
lightning/src/util/macro_logger.rs
lightning/src/util/ser.rs
lightning/src/util/ser_macros.rs
lightning/src/util/test_utils.rs

index a23f9d76a957b61fe5be5cb28f23f542b317286e..24a8c96ffaa4e94d88139a31549be0d4c079e0ec 100644 (file)
@@ -1,3 +1,105 @@
+# 0.0.101 - 2021-09-23
+
+## API Updates
+ * Custom message types are now supported directly in the `PeerManager`,
+   allowing you to send and receive messages of any type that is not natively
+   understood by LDK. This requires a new type bound on `PeerManager`, a
+   `CustomMessageHandler`. `IgnoringMessageHandler` provides a simple default
+   for this new bound for ignoring unknown messages (#1031, #1074).
+ * Route graph updates as a result of failed payments are no longer provided as
+   `MessageSendEvent::PaymentFailureNetworkUpdate` but instead included in a
+   new field in the `Event::PaymentFailed` events. Generally, this means route
+   graph updates are no longer handled as a part of the `PeerManager` but
+   instead through the new `EventHandler` implementation for
+   `NetGraphMsgHandler`. To make this easy, a new parameter to
+   `lightning-background-processor::BackgroundProcessor::start` is added, which
+   contains an `Option`al `NetGraphmsgHandler`. If provided as `Some`, relevant
+   events will be processed by the `NetGraphMsgHandler` prior to normal event
+   handling (#1043).
+ * `NetworkGraph` is now, itself, thread-safe. Accordingly, most functions now
+   take `&self` instead of `&mut self` and the graph data can be accessed
+   through `NetworkGraph.read_only` (#1043).
+ * The balances available on-chain to claim after a channel has been closed are
+   now exposed via `ChannelMonitor::get_claimable_balances` and
+   `ChainMonitor::get_claimable_balances`. The second can be used to get
+   information about all closed channels which still have on-chain balances
+   associated with them. See enum variants of `ln::channelmonitor::Balance` and
+   method documentation for the above methods for more information on the types
+   of balances exposed (#1034).
+ * When one HTLC of a multi-path payment fails, the new field `all_paths_failed`
+   in `Event::PaymentFailed` is set to `false`. This implies that the payment
+   has not failed, but only one part. Payment resolution is only indicated by an
+   `Event::PaymentSent` event or an `Event::PaymentFailed` with
+   `all_paths_failed` set to `true`, which is also set for the last remaining
+   part of a multi-path payment (#1053).
+ * To better capture the context described above, `Event::PaymentFailed` has
+   been renamed to `Event::PaymentPathFailed` (#1084).
+ * A new event, `ChannelClosed`, is provided by `ChannelManager` when a channel
+   is closed, including a reason and error message (if relevant, #997).
+ * `lightning-invoice` now considers invoices with sub-millisatoshi precision
+   to be invalid, and requires millisatoshi values during construction (thus
+   you must call `amount_milli_satoshis` instead of `amount_pico_btc`, #1057).
+ * The `BaseSign` interface now includes two new hooks which provide additional
+   information about commitment transaction signatures and revocation secrets
+   provided by our counterparty, allowing additional verification (#1039).
+ * The `BaseSign` interface now includes additional information for cooperative
+   close transactions, making it easier for a signer to verify requests (#1064).
+ * `Route` has two additional helper methods to get fees and amounts (#1063).
+ * `Txid` and `Transaction` objects can now be deserialized from responses when
+   using the HTTP client in the `lightning-block-sync` crate (#1037, #1061).
+
+## Bug Fixes
+ * Fix a panic when reading a lightning invoice with a non-recoverable
+   signature. Further, restrict lightning invoice parsing to require payment
+   secrets and better handle a few edge cases as required by BOLT 11 (#1057).
+ * Fix a panic when receiving multiple messages (such as HTLC fulfill messages)
+   after a call to `chain::Watch::update_channel` returned
+   `Err(ChannelMonitorUpdateErr::TemporaryFailure)` with no
+   `ChannelManager::channel_monitor_updated` call in between (#1066).
+ * For multi-path payments, `Event::PaymentSent` is no longer generated
+   multiple times, once for each independent part (#1053).
+ * Multi-hop route hints in invoices are now considered in the default router
+   provided via `get_route` (#1040).
+ * The time peers have to respond to pings has been increased when building
+   with debug assertions enabled. This avoids peer disconnections on slow hosts
+   when running in debug mode (#1051).
+ * The timeout for the first byte of a response for requests from the
+   `lightning-block-sync` crate has been increased to 300 seconds to better
+   handle the long hangs in Bitcoin Core when it syncs to disk (#1090).
+
+## Serialization Compatibility
+ * Due to a bug in 0.0.100, `Event`s written by 0.0.101 which are of a type not
+   understood by 0.0.100 may lead to `Err(DecodeError::InvalidValue)` or corrupt
+   deserialized objects in 0.100. Such `Event`s will lead to an
+   `Err(DecodeError::InvalidValue)` in versions prior to 0.0.100. The only such
+   new event written by 0.0.101 is `Event::ChannelClosed` (#1087).
+ * Payments that were initiated in versions prior to 0.0.101 may still
+   generate duplicate `PaymentSent` `Event`s or may have spurious values for
+   `Event::PaymentPathFailed::all_paths_failed` (#1053).
+ * The return values of `ChannelMonitor::get_claimable_balances` (and, thus,
+   `ChainMonitor::get_claimable_balances`) may be spurious for channels where
+   the spend of the funding transaction appeared on chain while running a
+   version prior to 0.0.101. `Balance` information should only be relied upon
+   for channels that were closed while running 0.0.101+ (#1034).
+ * Payments failed while running versions prior to 0.0.101 will never have a
+   `Some` for the `network_update` field (#1043).
+
+In total, this release features 67 files changed, 4980 insertions, 1888
+deletions in 89 commits from 12 authors, in alphabetical order:
+ * Antoine Riard
+ * Devrandom
+ * Galder Zamarreño
+ * Giles Cope
+ * Jeffrey Czyz
+ * Joseph Goulden
+ * Matt Corallo
+ * Sergi Delgado Segura
+ * Tibo-lg
+ * Valentine Wallace
+ * abhik-99
+ * vss96
+
+
 # 0.0.100 - 2021-08-17
 
 ## API Updates
index f59418cfd0c776ab79025ef937eb364126c3581e..dfa90fc0f979aa9d5f41de43e250140183939ec9 100644 (file)
@@ -80,6 +80,7 @@ mkdir -p ./test_cases/$TARGET
 echo $HEX | xxd -r -p > ./test_cases/$TARGET/any_filename_works
 
 export RUST_BACKTRACE=1
+export RUSTFLAGS="--cfg=fuzzing"
 cargo test
 ```
 
index 88826d65570fb94c7cfda47de7cfa58c53e5027e..4177b50275931722565fc1468aa7e5b08f29d76d 100644 (file)
@@ -94,9 +94,6 @@ impl Writer for VecWriter {
                self.0.extend_from_slice(buf);
                Ok(())
        }
-       fn size_hint(&mut self, size: usize) {
-               self.0.reserve_exact(size);
-       }
 }
 
 struct TestChainMonitor {
@@ -594,7 +591,6 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                },
                                                events::MessageSendEvent::SendFundingLocked { .. } => continue,
                                                events::MessageSendEvent::SendAnnouncementSignatures { .. } => continue,
-                                               events::MessageSendEvent::PaymentFailureNetworkUpdate { .. } => continue,
                                                events::MessageSendEvent::SendChannelUpdate { ref node_id, ref msg } => {
                                                        assert_eq!(msg.contents.flags & 2, 0); // The disable bit must never be set!
                                                        if Some(*node_id) == expect_drop_id { panic!("peer_disconnected should drop msgs bound for the disconnected peer"); }
@@ -727,10 +723,6 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                events::MessageSendEvent::SendAnnouncementSignatures { .. } => {
                                                        // Can be generated as a reestablish response
                                                },
-                                               events::MessageSendEvent::PaymentFailureNetworkUpdate { .. } => {
-                                                       // Can be generated due to a payment forward being rejected due to a
-                                                       // channel having previously failed a monitor update
-                                               },
                                                events::MessageSendEvent::SendChannelUpdate { ref msg, .. } => {
                                                        // When we reconnect we will resend a channel_update to make sure our
                                                        // counterparty has the latest parameters for receiving payments
@@ -769,7 +761,6 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                        events::MessageSendEvent::SendChannelReestablish { .. } => {},
                                                        events::MessageSendEvent::SendFundingLocked { .. } => {},
                                                        events::MessageSendEvent::SendAnnouncementSignatures { .. } => {},
-                                                       events::MessageSendEvent::PaymentFailureNetworkUpdate { .. } => {},
                                                        events::MessageSendEvent::SendChannelUpdate { ref msg, .. } => {
                                                                assert_eq!(msg.contents.flags & 2, 0); // The disable bit must never be set!
                                                        },
@@ -787,7 +778,6 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                        events::MessageSendEvent::SendChannelReestablish { .. } => {},
                                                        events::MessageSendEvent::SendFundingLocked { .. } => {},
                                                        events::MessageSendEvent::SendAnnouncementSignatures { .. } => {},
-                                                       events::MessageSendEvent::PaymentFailureNetworkUpdate { .. } => {},
                                                        events::MessageSendEvent::SendChannelUpdate { ref msg, .. } => {
                                                                assert_eq!(msg.contents.flags & 2, 0); // The disable bit must never be set!
                                                        },
@@ -837,7 +827,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
                                                        }
                                                },
                                                events::Event::PaymentSent { .. } => {},
-                                               events::Event::PaymentFailed { .. } => {},
+                                               events::Event::PaymentPathFailed { .. } => {},
                                                events::Event::PaymentForwarded { .. } if $node == 1 => {},
                                                events::Event::PendingHTLCsForwardable { .. } => {
                                                        nodes[$node].process_pending_htlc_forwards();
index 933930cf6d163148cb28832fde7cbcf47133b90b..c31930342f43101e2b4e4bbac717939e3047a677 100644 (file)
@@ -18,9 +18,6 @@ impl Writer for VecWriter {
                self.0.extend_from_slice(buf);
                Ok(())
        }
-       fn size_hint(&mut self, size: usize) {
-               self.0.reserve_exact(size);
-       }
 }
 
 #[inline]
index 28592ffda512fe6677a6431ebf35303526b2a648..799c2eb8cb94b2f3a1bdbf2abcfc9c43809f4fd6 100644 (file)
@@ -38,7 +38,7 @@ use lightning::ln::peer_handler::{MessageHandler,PeerManager,SocketDescriptor,Ig
 use lightning::ln::msgs::DecodeError;
 use lightning::ln::script::ShutdownScript;
 use lightning::routing::router::get_route;
-use lightning::routing::network_graph::NetGraphMsgHandler;
+use lightning::routing::network_graph::{NetGraphMsgHandler, NetworkGraph};
 use lightning::util::config::UserConfig;
 use lightning::util::errors::APIError;
 use lightning::util::events::Event;
@@ -378,7 +378,8 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
        };
        let channelmanager = Arc::new(ChannelManager::new(fee_est.clone(), monitor.clone(), broadcast.clone(), Arc::clone(&logger), keys_manager.clone(), config, params));
        let our_id = PublicKey::from_secret_key(&Secp256k1::signing_only(), &keys_manager.get_node_secret());
-       let net_graph_msg_handler = Arc::new(NetGraphMsgHandler::new(genesis_block(network).block_hash(), None, Arc::clone(&logger)));
+       let network_graph = NetworkGraph::new(genesis_block(network).block_hash());
+       let net_graph_msg_handler = Arc::new(NetGraphMsgHandler::new(network_graph, None, Arc::clone(&logger)));
 
        let peers = RefCell::new([false; 256]);
        let mut loss_detector = MoneyLossDetector::new(&peers, channelmanager.clone(), monitor.clone(), PeerManager::new(MessageHandler {
@@ -434,7 +435,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                        },
                        4 => {
                                let value = slice_to_be24(get_slice!(3)) as u64;
-                               let route = match get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &get_pubkey!(), None, None, &Vec::new(), value, 42, Arc::clone(&logger)) {
+                               let route = match get_route(&our_id, &net_graph_msg_handler.network_graph, &get_pubkey!(), None, None, &Vec::new(), value, 42, Arc::clone(&logger)) {
                                        Ok(route) => route,
                                        Err(_) => return,
                                };
@@ -451,7 +452,7 @@ pub fn do_test(data: &[u8], logger: &Arc<dyn Logger>) {
                        },
                        15 => {
                                let value = slice_to_be24(get_slice!(3)) as u64;
-                               let mut route = match get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &get_pubkey!(), None, None, &Vec::new(), value, 42, Arc::clone(&logger)) {
+                               let mut route = match get_route(&our_id, &net_graph_msg_handler.network_graph, &get_pubkey!(), None, None, &Vec::new(), value, 42, Arc::clone(&logger)) {
                                        Ok(route) => route,
                                        Err(_) => return,
                                };
@@ -679,7 +680,7 @@ mod tests {
                // 030012 - inbound read from peer id 0 of len 18
                // 0141 03000000000000000000000000000000 - message header indicating message length 321
                // 0300fe - inbound read from peer id 0 of len 254
-               // 0020 7500000000000000000000000000000000000000000000000000000000000000 ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679 000000000000c350 0000000000000000 000000000000014a ffffffffffffffff 0000000000000222 0000000000000000 000000fd 0006 01e3 030000000000000000000000000000000000000000000000000000000000000001 030000000000000000000000000000000000000000000000000000000000000002 030000000000000000000000000000000000000000000000000000000000000003 030000000000000000000000000000000000000000000000000000000000000004 - beginning of open_channel message
+               // 0020 7500000000000000000000000000000000000000000000000000000000000000 ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679 000000000000c350 0000000000000000 0000000000000162 ffffffffffffffff 0000000000000222 0000000000000000 000000fd 0006 01e3 030000000000000000000000000000000000000000000000000000000000000001 030000000000000000000000000000000000000000000000000000000000000002 030000000000000000000000000000000000000000000000000000000000000003 030000000000000000000000000000000000000000000000000000000000000004 - beginning of open_channel message
                // 030053 - inbound read from peer id 0 of len 83
                // 030000000000000000000000000000000000000000000000000000000000000005 020900000000000000000000000000000000000000000000000000000000000000 01 03000000000000000000000000000000 - rest of open_channel and mac
                //
@@ -730,7 +731,7 @@ mod tests {
                // 030112 - inbound read from peer id 1 of len 18
                // 0110 01000000000000000000000000000000 - message header indicating message length 272
                // 0301ff - inbound read from peer id 1 of len 255
-               // 0021 0000000000000000000000000000000000000000000000000000000000000e05 000000000000014a 00000000004c4b40 00000000000003e8 00000000000003e8 00000002 03f0 0005 030000000000000000000000000000000000000000000000000000000000000100 030000000000000000000000000000000000000000000000000000000000000200 030000000000000000000000000000000000000000000000000000000000000300 030000000000000000000000000000000000000000000000000000000000000400 030000000000000000000000000000000000000000000000000000000000000500 02660000000000000000000000000000 - beginning of accept_channel
+               // 0021 0000000000000000000000000000000000000000000000000000000000000e05 0000000000000162 00000000004c4b40 00000000000003e8 00000000000003e8 00000002 03f0 0005 030000000000000000000000000000000000000000000000000000000000000100 030000000000000000000000000000000000000000000000000000000000000200 030000000000000000000000000000000000000000000000000000000000000300 030000000000000000000000000000000000000000000000000000000000000400 030000000000000000000000000000000000000000000000000000000000000500 02660000000000000000000000000000 - beginning of accept_channel
                // 030121 - inbound read from peer id 1 of len 33
                // 0000000000000000000000000000000000 01000000000000000000000000000000 - rest of accept_channel and mac
                //
@@ -926,7 +927,7 @@ mod tests {
                // - client now fails the HTLC backwards as it was unable to extract the payment preimage (CHECK 9 duplicate and CHECK 10)
 
                let logger = Arc::new(TrackingLogger { lines: Mutex::new(HashMap::new()) });
-               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c3500000000000000000000000000000014affffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000014a00000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4b000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
+               super::do_test(&::hex::decode("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012000a0300000000000000000000000000000003001a00100002200000022000030000000000000000000000000000000300120141030000000000000000000000000000000300fe00207500000000000000000000000000000000000000000000000000000000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030053030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010300000000000000000000000000000000fd00fd00fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112000a0100000000000000000000000000000003011a0010000220000002200001000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120110010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f00005030000000000000000000000000000000000000000000000000000000000000100030000000000000000000000000000000000000000000000000000000000000200030000000000000000000000000000000000000000000000000000000000000300030000000000000000000000000000000000000000000000000000000000000400030000000000000000000000000000000000000000000000000000000000000500026600000000000000000000000000000301210000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000003e8000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4e000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005550000000e000001000000000000000927c0000000a00000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff4b000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc<dyn Logger>));
 
                let log_entries = logger.lines.lock().unwrap();
                assert_eq!(log_entries.get(&("lightning::ln::peer_handler".to_string(), "Handling SendAcceptChannel event in peer_handler for node 030000000000000000000000000000000000000000000000000000000000000002 for channel ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679".to_string())), Some(&1)); // 1
index 0c1d061a74dc6047a334b386c05056349c04747e..2e1334841383e5f5b022f3d131338689bf823b9a 100755 (executable)
@@ -11,36 +11,36 @@ echo "mod utils;" > mod.rs
 
 # Note when adding new targets here you should add a similar line in src/bin/gen_target.sh
 
-GEN_TEST AcceptChannel test_msg ""
-GEN_TEST AnnouncementSignatures test_msg ""
+GEN_TEST AcceptChannel test_msg_simple ""
+GEN_TEST AnnouncementSignatures test_msg_simple ""
+GEN_TEST ClosingSigned test_msg_simple ""
+GEN_TEST CommitmentSigned test_msg_simple ""
+GEN_TEST FundingCreated test_msg_simple ""
+GEN_TEST FundingLocked test_msg_simple ""
+GEN_TEST FundingSigned test_msg_simple ""
+GEN_TEST GossipTimestampFilter test_msg_simple ""
+GEN_TEST Init test_msg_simple ""
+GEN_TEST OnionHopData test_msg_simple ""
+GEN_TEST OpenChannel test_msg_simple ""
+GEN_TEST Ping test_msg_simple ""
+GEN_TEST Pong test_msg_simple ""
+GEN_TEST QueryChannelRange test_msg_simple ""
+GEN_TEST ReplyShortChannelIdsEnd test_msg_simple ""
+GEN_TEST RevokeAndACK test_msg_simple ""
+GEN_TEST Shutdown test_msg_simple ""
+GEN_TEST UpdateAddHTLC test_msg_simple ""
+GEN_TEST UpdateFailHTLC test_msg_simple ""
+GEN_TEST UpdateFailMalformedHTLC test_msg_simple ""
+GEN_TEST UpdateFee test_msg_simple ""
+GEN_TEST UpdateFulfillHTLC test_msg_simple ""
+
 GEN_TEST ChannelReestablish test_msg ""
-GEN_TEST CommitmentSigned test_msg ""
 GEN_TEST DecodedOnionErrorPacket test_msg ""
-GEN_TEST FundingCreated test_msg ""
-GEN_TEST FundingLocked test_msg ""
-GEN_TEST FundingSigned test_msg ""
-GEN_TEST OpenChannel test_msg ""
-GEN_TEST RevokeAndACK test_msg ""
-GEN_TEST Shutdown test_msg ""
-GEN_TEST UpdateFailHTLC test_msg ""
-GEN_TEST UpdateFailMalformedHTLC test_msg ""
-GEN_TEST UpdateFee test_msg ""
-GEN_TEST UpdateFulfillHTLC test_msg ""
 
 GEN_TEST ChannelAnnouncement test_msg_exact ""
 GEN_TEST NodeAnnouncement test_msg_exact ""
 GEN_TEST QueryShortChannelIds test_msg ""
-GEN_TEST ReplyShortChannelIdsEnd test_msg ""
-GEN_TEST QueryChannelRange test_msg ""
 GEN_TEST ReplyChannelRange test_msg ""
-GEN_TEST GossipTimestampFilter test_msg ""
 
-GEN_TEST UpdateAddHTLC test_msg_hole ", 85, 33"
 GEN_TEST ErrorMessage test_msg_hole ", 32, 2"
 GEN_TEST ChannelUpdate test_msg_hole ", 108, 1"
-
-GEN_TEST ClosingSigned test_msg_simple ""
-GEN_TEST Init test_msg_simple ""
-GEN_TEST OnionHopData test_msg_simple ""
-GEN_TEST Ping test_msg_simple ""
-GEN_TEST Pong test_msg_simple ""
index 0f273cb7606d1e51b5304e5cfe46671c441e2b5f..af70c58e30ea2f9e9c5bfe330372cfe01948a8ce 100644 (file)
@@ -1,31 +1,31 @@
 mod utils;
 pub mod msg_accept_channel;
 pub mod msg_announcement_signatures;
-pub mod msg_channel_reestablish;
+pub mod msg_closing_signed;
 pub mod msg_commitment_signed;
-pub mod msg_decoded_onion_error_packet;
 pub mod msg_funding_created;
 pub mod msg_funding_locked;
 pub mod msg_funding_signed;
+pub mod msg_gossip_timestamp_filter;
+pub mod msg_init;
+pub mod msg_onion_hop_data;
 pub mod msg_open_channel;
+pub mod msg_ping;
+pub mod msg_pong;
+pub mod msg_query_channel_range;
+pub mod msg_reply_short_channel_ids_end;
 pub mod msg_revoke_and_ack;
 pub mod msg_shutdown;
+pub mod msg_update_add_htlc;
 pub mod msg_update_fail_htlc;
 pub mod msg_update_fail_malformed_htlc;
 pub mod msg_update_fee;
 pub mod msg_update_fulfill_htlc;
+pub mod msg_channel_reestablish;
+pub mod msg_decoded_onion_error_packet;
 pub mod msg_channel_announcement;
 pub mod msg_node_announcement;
 pub mod msg_query_short_channel_ids;
-pub mod msg_reply_short_channel_ids_end;
-pub mod msg_query_channel_range;
 pub mod msg_reply_channel_range;
-pub mod msg_gossip_timestamp_filter;
-pub mod msg_update_add_htlc;
 pub mod msg_error_message;
 pub mod msg_channel_update;
-pub mod msg_closing_signed;
-pub mod msg_init;
-pub mod msg_onion_hop_data;
-pub mod msg_ping;
-pub mod msg_pong;
index 095556d57d9e6fe4c9323b13d5ab6c4a49d8a1d0..a8ec438784bca1a4c6d4fae28d5d9fd2b735ce5f 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_accept_channel_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::AcceptChannel, data);
+       test_msg_simple!(msgs::AcceptChannel, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_accept_channel_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::AcceptChannel, data);
+       test_msg_simple!(msgs::AcceptChannel, data);
 }
index 138ab5dbf190392368d23b31f85b626143cfb656..0fc40fcb69bd7353a1a261370f81fccf166c3c32 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_announcement_signatures_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::AnnouncementSignatures, data);
+       test_msg_simple!(msgs::AnnouncementSignatures, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_announcement_signatures_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::AnnouncementSignatures, data);
+       test_msg_simple!(msgs::AnnouncementSignatures, data);
 }
index 9ac846f6f100194e272fe1ebef4c3adf08a83aea..163ce74460678cc6365cc2f94ccf11b70252e31d 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_commitment_signed_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::CommitmentSigned, data);
+       test_msg_simple!(msgs::CommitmentSigned, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_commitment_signed_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::CommitmentSigned, data);
+       test_msg_simple!(msgs::CommitmentSigned, data);
 }
index 3b61a64bf6a9f8b2d98ddfb0f686845f15a8c0ae..e0005cb082a4523d181fb94b344afe900a86d8fb 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_created_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::FundingCreated, data);
+       test_msg_simple!(msgs::FundingCreated, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_created_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::FundingCreated, data);
+       test_msg_simple!(msgs::FundingCreated, data);
 }
index 677af731895a847bf26c1b34da05680a3c6d9143..2c6ad63c96375dae55c448d862fec1e33389f0ed 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_locked_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::FundingLocked, data);
+       test_msg_simple!(msgs::FundingLocked, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_locked_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::FundingLocked, data);
+       test_msg_simple!(msgs::FundingLocked, data);
 }
index 388738ff2d480634a589fae57ec473bfc2bcd7c2..f0586e7b294a06b48200f304c35c51eb98160e51 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_funding_signed_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::FundingSigned, data);
+       test_msg_simple!(msgs::FundingSigned, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_funding_signed_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::FundingSigned, data);
+       test_msg_simple!(msgs::FundingSigned, data);
 }
index 73999cd3a86a403a1d51d6ab5b7e630af3fc74d8..448aaffc90c2b88432500cf6b08ea5143e0d61e1 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_gossip_timestamp_filter_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::GossipTimestampFilter, data);
+       test_msg_simple!(msgs::GossipTimestampFilter, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_gossip_timestamp_filter_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::GossipTimestampFilter, data);
+       test_msg_simple!(msgs::GossipTimestampFilter, data);
 }
index b0e96734e4c8f3706d948374e66f2ef2bc8b46a2..ce637c167efe20f83253ca7b5c39d969e5c88441 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_open_channel_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::OpenChannel, data);
+       test_msg_simple!(msgs::OpenChannel, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_open_channel_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::OpenChannel, data);
+       test_msg_simple!(msgs::OpenChannel, data);
 }
index bd16147f6417c2805493ec92edec77aac744f646..4b3de6aa895b151a6817acdbff939ac4598329ee 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_query_channel_range_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::QueryChannelRange, data);
+       test_msg_simple!(msgs::QueryChannelRange, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_query_channel_range_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::QueryChannelRange, data);
+       test_msg_simple!(msgs::QueryChannelRange, data);
 }
index fcd13154370b729b42966a917892847e8ff021e9..7634329a435a7d995aa3a52fd87e11744b8b6369 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_reply_short_channel_ids_end_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::ReplyShortChannelIdsEnd, data);
+       test_msg_simple!(msgs::ReplyShortChannelIdsEnd, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_reply_short_channel_ids_end_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::ReplyShortChannelIdsEnd, data);
+       test_msg_simple!(msgs::ReplyShortChannelIdsEnd, data);
 }
index 41f40becdfd0e7fdab951ef5cf88463290670b07..873939ca7c9d85b34c1d8cf15eb33f707adaec77 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_revoke_and_ack_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::RevokeAndACK, data);
+       test_msg_simple!(msgs::RevokeAndACK, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_revoke_and_ack_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::RevokeAndACK, data);
+       test_msg_simple!(msgs::RevokeAndACK, data);
 }
index 3e5358205cd6cc9cc50db92da5957a81c3dd1bcb..e6e74cc661a9c5c586cb98793281a42f7a3e1fd7 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_shutdown_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::Shutdown, data);
+       test_msg_simple!(msgs::Shutdown, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_shutdown_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::Shutdown, data);
+       test_msg_simple!(msgs::Shutdown, data);
 }
index d45eeadb19490f8a2ca92618fa93449a33a72e97..409f0fac8dff62e7f097ba6fad7a279230c29181 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_add_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg_hole!(msgs::UpdateAddHTLC, data, 85, 33);
+       test_msg_simple!(msgs::UpdateAddHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_add_htlc_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg_hole!(msgs::UpdateAddHTLC, data, 85, 33);
+       test_msg_simple!(msgs::UpdateAddHTLC, data);
 }
index 8fadde21030fe48f90e832b9d6609f29b1ce2a01..12d3f1c3fc3f0697d0cb0b5f130afa566fa7fca6 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fail_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::UpdateFailHTLC, data);
+       test_msg_simple!(msgs::UpdateFailHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fail_htlc_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::UpdateFailHTLC, data);
+       test_msg_simple!(msgs::UpdateFailHTLC, data);
 }
index 9d0481481bcd03042734f79bb9b981d53a5d4339..0b36070d1e59d6d5076ef6e4579c1ba92290cc25 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fail_malformed_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::UpdateFailMalformedHTLC, data);
+       test_msg_simple!(msgs::UpdateFailMalformedHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fail_malformed_htlc_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::UpdateFailMalformedHTLC, data);
+       test_msg_simple!(msgs::UpdateFailMalformedHTLC, data);
 }
index 318f5b089eb6d312fee4e71aaac4a81bab56433d..3c23a902f5723dab1fef5cc3fe2e7a01aee2a52b 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fee_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::UpdateFee, data);
+       test_msg_simple!(msgs::UpdateFee, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fee_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::UpdateFee, data);
+       test_msg_simple!(msgs::UpdateFee, data);
 }
index 692cb972f7817c3d09b29e2a29c3e7e9c521cfdd..460ff0e167880a6771804c0a19213233575ef4c8 100644 (file)
@@ -17,11 +17,11 @@ use utils::test_logger;
 
 #[inline]
 pub fn msg_update_fulfill_htlc_test<Out: test_logger::Output>(data: &[u8], _out: Out) {
-       test_msg!(msgs::UpdateFulfillHTLC, data);
+       test_msg_simple!(msgs::UpdateFulfillHTLC, data);
 }
 
 #[no_mangle]
 pub extern "C" fn msg_update_fulfill_htlc_run(data: *const u8, datalen: usize) {
        let data = unsafe { std::slice::from_raw_parts(data, datalen) };
-       test_msg!(msgs::UpdateFulfillHTLC, data);
+       test_msg_simple!(msgs::UpdateFulfillHTLC, data);
 }
index 82fa739fbcfbd4febfb33af89b6bbcb31717d559..6325f3bf10d07bab3469b018ab106c7c732ec1d7 100644 (file)
@@ -13,13 +13,9 @@ use lightning::util::ser::Writer;
 pub struct VecWriter(pub Vec<u8>);
 impl Writer for VecWriter {
        fn write_all(&mut self, buf: &[u8]) -> Result<(), ::std::io::Error> {
-               assert!(self.0.capacity() >= self.0.len() + buf.len());
                self.0.extend_from_slice(buf);
                Ok(())
        }
-       fn size_hint(&mut self, size: usize) {
-               self.0.reserve_exact(size);
-       }
 }
 
 // We attempt to test the strictest behavior we can for a given message, however, some messages
@@ -43,6 +39,7 @@ macro_rules! test_msg {
                                msg.write(&mut w).unwrap();
 
                                assert_eq!(w.0.len(), p);
+                               assert_eq!(msg.serialized_length(), p);
                                assert_eq!(&r.into_inner()[..p], &w.0[..p]);
                        }
                }
@@ -60,6 +57,7 @@ macro_rules! test_msg_simple {
                        if let Ok(msg) = <$MsgType as Readable>::read(&mut r) {
                                let mut w = VecWriter(Vec::new());
                                msg.write(&mut w).unwrap();
+                               assert_eq!(msg.serialized_length(), w.0.len());
 
                                let msg = <$MsgType as Readable>::read(&mut ::std::io::Cursor::new(&w.0)).unwrap();
                                let mut w_two = VecWriter(Vec::new());
@@ -82,6 +80,7 @@ macro_rules! test_msg_exact {
                                let mut w = VecWriter(Vec::new());
                                msg.write(&mut w).unwrap();
                                assert_eq!(&r.into_inner()[..], &w.0[..]);
+                               assert_eq!(msg.serialized_length(), w.0.len());
                        }
                }
        }
@@ -99,6 +98,7 @@ macro_rules! test_msg_hole {
                                let mut w = VecWriter(Vec::new());
                                msg.write(&mut w).unwrap();
                                let p = w.0.len() as usize;
+                               assert_eq!(msg.serialized_length(), p);
 
                                assert_eq!(w.0.len(), p);
                                assert_eq!(&r.get_ref()[..$hole], &w.0[..$hole]);
index baa32312ee9b769aef249810f80894c300507ac8..6c792916f8215f40efc9d5a8a66e403ce215eaf9 100644 (file)
@@ -160,7 +160,7 @@ pub fn do_test<Out: test_logger::Output>(data: &[u8], out: Out) {
        let logger: Arc<dyn Logger> = Arc::new(test_logger::TestLogger::new("".to_owned(), out));
 
        let our_pubkey = get_pubkey!();
-       let mut net_graph = NetworkGraph::new(genesis_block(Network::Bitcoin).header.block_hash());
+       let net_graph = NetworkGraph::new(genesis_block(Network::Bitcoin).header.block_hash());
 
        let mut node_pks = HashSet::new();
        let mut scid = 42;
index 98d9ade945e5222bca928daa6739c060d60ad1f8..178ffba80403acbffc35e4e7cb127e039fbe5a43 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-background-processor"
-version = "0.0.100"
+version = "0.0.101"
 authors = ["Valentine Wallace <vwallace@protonmail.com>"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/rust-bitcoin/rust-lightning"
@@ -11,9 +11,9 @@ edition = "2018"
 
 [dependencies]
 bitcoin = "0.27"
-lightning = { version = "0.0.100", path = "../lightning", features = ["allow_wallclock_use"] }
-lightning-persister = { version = "0.0.100", path = "../lightning-persister" }
+lightning = { version = "0.0.101", path = "../lightning", features = ["allow_wallclock_use"] }
+lightning-persister = { version = "0.0.101", path = "../lightning-persister" }
 
 [dev-dependencies]
-lightning = { version = "0.0.100", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.101", path = "../lightning", features = ["_test_utils"] }
 
index 34cddd1a6aa8b83ea36699e0bb39c4a21b97480a..c7d55ae61b7ecdc9ebb586654efd76a0dd650397 100644 (file)
@@ -17,7 +17,8 @@ use lightning::ln::channelmanager::ChannelManager;
 use lightning::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler};
 use lightning::ln::peer_handler::{PeerManager, SocketDescriptor};
 use lightning::ln::peer_handler::CustomMessageHandler;
-use lightning::util::events::{EventHandler, EventsProvider};
+use lightning::routing::network_graph::NetGraphMsgHandler;
+use lightning::util::events::{Event, EventHandler, EventsProvider};
 use lightning::util::logger::Logger;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, Ordering};
@@ -26,20 +27,28 @@ use std::thread::JoinHandle;
 use std::time::{Duration, Instant};
 use std::ops::Deref;
 
-/// BackgroundProcessor takes care of tasks that (1) need to happen periodically to keep
+/// `BackgroundProcessor` takes care of tasks that (1) need to happen periodically to keep
 /// Rust-Lightning running properly, and (2) either can or should be run in the background. Its
 /// responsibilities are:
-/// * Monitoring whether the ChannelManager needs to be re-persisted to disk, and if so,
+/// * Processing [`Event`]s with a user-provided [`EventHandler`].
+/// * Monitoring whether the [`ChannelManager`] needs to be re-persisted to disk, and if so,
 ///   writing it to disk/backups by invoking the callback given to it at startup.
-///   ChannelManager persistence should be done in the background.
-/// * Calling `ChannelManager::timer_tick_occurred()` and
-///   `PeerManager::timer_tick_occurred()` every minute (can be done in the
-///   background).
+///   [`ChannelManager`] persistence should be done in the background.
+/// * Calling [`ChannelManager::timer_tick_occurred`] and [`PeerManager::timer_tick_occurred`]
+///   at the appropriate intervals.
 ///
-/// Note that if ChannelManager persistence fails and the persisted manager becomes out-of-date,
-/// then there is a risk of channels force-closing on startup when the manager realizes it's
-/// outdated. However, as long as `ChannelMonitor` backups are sound, no funds besides those used
-/// for unilateral chain closure fees are at risk.
+/// It will also call [`PeerManager::process_events`] periodically though this shouldn't be relied
+/// upon as doing so may result in high latency.
+///
+/// # Note
+///
+/// If [`ChannelManager`] persistence fails and the persisted manager becomes out-of-date, then
+/// there is a risk of channels force-closing on startup when the manager realizes it's outdated.
+/// However, as long as [`ChannelMonitor`] backups are sound, no funds besides those used for
+/// unilateral chain closure fees are at risk.
+///
+/// [`ChannelMonitor`]: lightning::chain::channelmonitor::ChannelMonitor
+/// [`Event`]: lightning::util::events::Event
 #[must_use = "BackgroundProcessor will immediately stop on drop. It should be stored until shutdown."]
 pub struct BackgroundProcessor {
        stop_thread: Arc<AtomicBool>,
@@ -91,6 +100,33 @@ ChannelManagerPersister<Signer, M, T, K, F, L> for Fun where
        }
 }
 
+/// Decorates an [`EventHandler`] with common functionality provided by standard [`EventHandler`]s.
+struct DecoratingEventHandler<
+       E: EventHandler,
+       N: Deref<Target = NetGraphMsgHandler<A, L>>,
+       A: Deref,
+       L: Deref,
+>
+where A::Target: chain::Access, L::Target: Logger {
+       event_handler: E,
+       net_graph_msg_handler: Option<N>,
+}
+
+impl<
+       E: EventHandler,
+       N: Deref<Target = NetGraphMsgHandler<A, L>>,
+       A: Deref,
+       L: Deref,
+> EventHandler for DecoratingEventHandler<E, N, A, L>
+where A::Target: chain::Access, L::Target: Logger {
+       fn handle_event(&self, event: &Event) {
+               if let Some(event_handler) = &self.net_graph_msg_handler {
+                       event_handler.handle_event(event);
+               }
+               self.event_handler.handle_event(event);
+       }
+}
+
 impl BackgroundProcessor {
        /// Start a background thread that takes care of responsibilities enumerated in the [top-level
        /// documentation].
@@ -99,23 +135,34 @@ impl BackgroundProcessor {
        /// `persist_manager` returns an error. In case of an error, the error is retrieved by calling
        /// either [`join`] or [`stop`].
        ///
-       /// Typically, users should either implement [`ChannelManagerPersister`] to never return an
-       /// error or call [`join`] and handle any error that may arise. For the latter case, the
-       /// `BackgroundProcessor` must be restarted by calling `start` again after handling the error.
+       /// # Data Persistence
        ///
        /// `persist_manager` is responsible for writing out the [`ChannelManager`] to disk, and/or
        /// uploading to one or more backup services. See [`ChannelManager::write`] for writing out a
        /// [`ChannelManager`]. See [`FilesystemPersister::persist_manager`] for Rust-Lightning's
        /// provided implementation.
        ///
+       /// Typically, users should either implement [`ChannelManagerPersister`] to never return an
+       /// error or call [`join`] and handle any error that may arise. For the latter case,
+       /// `BackgroundProcessor` must be restarted by calling `start` again after handling the error.
+       ///
+       /// # Event Handling
+       ///
+       /// `event_handler` is responsible for handling events that users should be notified of (e.g.,
+       /// payment failed). [`BackgroundProcessor`] may decorate the given [`EventHandler`] with common
+       /// functionality implemented by other handlers.
+       /// * [`NetGraphMsgHandler`] if given will update the [`NetworkGraph`] based on payment failures.
+       ///
        /// [top-level documentation]: Self
        /// [`join`]: Self::join
        /// [`stop`]: Self::stop
        /// [`ChannelManager`]: lightning::ln::channelmanager::ChannelManager
        /// [`ChannelManager::write`]: lightning::ln::channelmanager::ChannelManager#impl-Writeable
        /// [`FilesystemPersister::persist_manager`]: lightning_persister::FilesystemPersister::persist_manager
+       /// [`NetworkGraph`]: lightning::routing::network_graph::NetworkGraph
        pub fn start<
                Signer: 'static + Sign,
+               CA: 'static + Deref + Send + Sync,
                CF: 'static + Deref + Send + Sync,
                CW: 'static + Deref + Send + Sync,
                T: 'static + Deref + Send + Sync,
@@ -130,11 +177,15 @@ impl BackgroundProcessor {
                CMP: 'static + Send + ChannelManagerPersister<Signer, CW, T, K, F, L>,
                M: 'static + Deref<Target = ChainMonitor<Signer, CF, T, F, L, P>> + Send + Sync,
                CM: 'static + Deref<Target = ChannelManager<Signer, CW, T, K, F, L>> + Send + Sync,
+               NG: 'static + Deref<Target = NetGraphMsgHandler<CA, L>> + Send + Sync,
                UMH: 'static + Deref + Send + Sync,
                PM: 'static + Deref<Target = PeerManager<Descriptor, CMH, RMH, L, UMH>> + Send + Sync,
-       >
-       (persister: CMP, event_handler: EH, chain_monitor: M, channel_manager: CM, peer_manager: PM, logger: L) -> Self
+       >(
+               persister: CMP, event_handler: EH, chain_monitor: M, channel_manager: CM,
+               net_graph_msg_handler: Option<NG>, peer_manager: PM, logger: L
+       ) -> Self
        where
+               CA::Target: 'static + chain::Access,
                CF::Target: 'static + chain::Filter,
                CW::Target: 'static + chain::Watch<Signer>,
                T::Target: 'static + BroadcasterInterface,
@@ -149,6 +200,8 @@ impl BackgroundProcessor {
                let stop_thread = Arc::new(AtomicBool::new(false));
                let stop_thread_clone = stop_thread.clone();
                let handle = thread::spawn(move || -> Result<(), std::io::Error> {
+                       let event_handler = DecoratingEventHandler { event_handler, net_graph_msg_handler };
+
                        log_trace!(logger, "Calling ChannelManager's timer_tick_occurred on startup");
                        channel_manager.timer_tick_occurred();
 
@@ -257,6 +310,7 @@ mod tests {
        use lightning::ln::features::InitFeatures;
        use lightning::ln::msgs::{ChannelMessageHandler, Init};
        use lightning::ln::peer_handler::{PeerManager, MessageHandler, SocketDescriptor, IgnoringMessageHandler};
+       use lightning::routing::network_graph::{NetworkGraph, NetGraphMsgHandler};
        use lightning::util::config::UserConfig;
        use lightning::util::events::{Event, MessageSendEventsProvider, MessageSendEvent};
        use lightning::util::ser::Writeable;
@@ -284,6 +338,7 @@ mod tests {
 
        struct Node {
                node: Arc<SimpleArcChannelManager<ChainMonitor, test_utils::TestBroadcaster, test_utils::TestFeeEstimator, test_utils::TestLogger>>,
+               net_graph_msg_handler: Option<Arc<NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>>>,
                peer_manager: Arc<PeerManager<TestDescriptor, Arc<test_utils::TestChannelMessageHandler>, Arc<test_utils::TestRoutingMessageHandler>, Arc<test_utils::TestLogger>, IgnoringMessageHandler>>,
                chain_monitor: Arc<ChainMonitor>,
                persister: Arc<FilesystemPersister>,
@@ -318,15 +373,18 @@ mod tests {
                        let persister = Arc::new(FilesystemPersister::new(format!("{}_persister_{}", persist_dir, i)));
                        let seed = [i as u8; 32];
                        let network = Network::Testnet;
-                       let now = Duration::from_secs(genesis_block(network).header.time as u64);
+                       let genesis_block = genesis_block(network);
+                       let now = Duration::from_secs(genesis_block.header.time as u64);
                        let keys_manager = Arc::new(KeysManager::new(&seed, now.as_secs(), now.subsec_nanos()));
                        let chain_monitor = Arc::new(chainmonitor::ChainMonitor::new(Some(chain_source.clone()), tx_broadcaster.clone(), logger.clone(), fee_estimator.clone(), persister.clone()));
                        let best_block = BestBlock::from_genesis(network);
                        let params = ChainParameters { network, best_block };
                        let manager = Arc::new(ChannelManager::new(fee_estimator.clone(), chain_monitor.clone(), tx_broadcaster.clone(), logger.clone(), keys_manager.clone(), UserConfig::default(), params));
+                       let network_graph = NetworkGraph::new(genesis_block.header.block_hash());
+                       let net_graph_msg_handler = Some(Arc::new(NetGraphMsgHandler::new(network_graph, Some(chain_source.clone()), logger.clone())));
                        let msg_handler = MessageHandler { chan_handler: Arc::new(test_utils::TestChannelMessageHandler::new()), route_handler: Arc::new(test_utils::TestRoutingMessageHandler::new() )};
                        let peer_manager = Arc::new(PeerManager::new(msg_handler, keys_manager.get_node_secret(), &seed, logger.clone(), IgnoringMessageHandler{}));
-                       let node = Node { node: manager, peer_manager, chain_monitor, persister, tx_broadcaster, logger, best_block };
+                       let node = Node { node: manager, net_graph_msg_handler, peer_manager, chain_monitor, persister, tx_broadcaster, logger, best_block };
                        nodes.push(node);
                }
 
@@ -345,7 +403,7 @@ mod tests {
                        begin_open_channel!($node_a, $node_b, $channel_value);
                        let events = $node_a.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
-                       let (temporary_channel_id, tx) = handle_funding_generation_ready!(events[0], $channel_value);
+                       let (temporary_channel_id, tx) = handle_funding_generation_ready!(&events[0], $channel_value);
                        end_open_channel!($node_a, $node_b, temporary_channel_id, tx);
                        tx
                }}
@@ -362,14 +420,14 @@ mod tests {
        macro_rules! handle_funding_generation_ready {
                ($event: expr, $channel_value: expr) => {{
                        match $event {
-                               Event::FundingGenerationReady { ref temporary_channel_id, ref channel_value_satoshis, ref output_script, user_channel_id } => {
-                                       assert_eq!(*channel_value_satoshis, $channel_value);
+                               &Event::FundingGenerationReady { temporary_channel_id, channel_value_satoshis, ref output_script, user_channel_id } => {
+                                       assert_eq!(channel_value_satoshis, $channel_value);
                                        assert_eq!(user_channel_id, 42);
 
                                        let tx = Transaction { version: 1 as i32, lock_time: 0, input: Vec::new(), output: vec![TxOut {
-                                               value: *channel_value_satoshis, script_pubkey: output_script.clone(),
+                                               value: channel_value_satoshis, script_pubkey: output_script.clone(),
                                        }]};
-                                       (*temporary_channel_id, tx)
+                                       (temporary_channel_id, tx)
                                },
                                _ => panic!("Unexpected event"),
                        }
@@ -423,8 +481,8 @@ mod tests {
                // Initiate the background processors to watch each node.
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = move |node: &ChannelManager<InMemorySigner, Arc<ChainMonitor>, Arc<test_utils::TestBroadcaster>, Arc<KeysManager>, Arc<test_utils::TestFeeEstimator>, Arc<test_utils::TestLogger>>| FilesystemPersister::persist_manager(data_dir.clone(), node);
-               let event_handler = |_| {};
-               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
+               let event_handler = |_: &_| {};
+               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].net_graph_msg_handler.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
 
                macro_rules! check_persisted_data {
                        ($node: expr, $filepath: expr, $expected_bytes: expr) => {
@@ -476,8 +534,8 @@ mod tests {
                let nodes = create_nodes(1, "test_timer_tick_called".to_string());
                let data_dir = nodes[0].persister.get_data_dir();
                let persister = move |node: &ChannelManager<InMemorySigner, Arc<ChainMonitor>, Arc<test_utils::TestBroadcaster>, Arc<KeysManager>, Arc<test_utils::TestFeeEstimator>, Arc<test_utils::TestLogger>>| FilesystemPersister::persist_manager(data_dir.clone(), node);
-               let event_handler = |_| {};
-               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
+               let event_handler = |_: &_| {};
+               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].net_graph_msg_handler.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
                loop {
                        let log_entries = nodes[0].logger.lines.lock().unwrap();
                        let desired_log = "Calling ChannelManager's timer_tick_occurred".to_string();
@@ -498,8 +556,8 @@ mod tests {
                open_channel!(nodes[0], nodes[1], 100000);
 
                let persister = |_: &_| Err(std::io::Error::new(std::io::ErrorKind::Other, "test"));
-               let event_handler = |_| {};
-               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
+               let event_handler = |_: &_| {};
+               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].net_graph_msg_handler.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
                match bg_processor.join() {
                        Ok(_) => panic!("Expected error persisting manager"),
                        Err(e) => {
@@ -518,10 +576,10 @@ mod tests {
 
                // Set up a background event handler for FundingGenerationReady events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
-               let event_handler = move |event| {
+               let event_handler = move |event: &Event| {
                        sender.send(handle_funding_generation_ready!(event, channel_value)).unwrap();
                };
-               let bg_processor = BackgroundProcessor::start(persister.clone(), event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
+               let bg_processor = BackgroundProcessor::start(persister.clone(), event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].net_graph_msg_handler.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
 
                // Open a channel and check that the FundingGenerationReady event was handled.
                begin_open_channel!(nodes[0], nodes[1], channel_value);
@@ -544,8 +602,8 @@ mod tests {
 
                // Set up a background event handler for SpendableOutputs events.
                let (sender, receiver) = std::sync::mpsc::sync_channel(1);
-               let event_handler = move |event| sender.send(event).unwrap();
-               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
+               let event_handler = move |event: &Event| sender.send(event.clone()).unwrap();
+               let bg_processor = BackgroundProcessor::start(persister, event_handler, nodes[0].chain_monitor.clone(), nodes[0].node.clone(), nodes[0].net_graph_msg_handler.clone(), nodes[0].peer_manager.clone(), nodes[0].logger.clone());
 
                // Force close the channel and check that the SpendableOutputs event was handled.
                nodes[0].node.force_close_channel(&nodes[0].node.list_channels()[0].channel_id).unwrap();
@@ -556,6 +614,7 @@ mod tests {
                        .expect("SpendableOutputs not handled within deadline");
                match event {
                        Event::SpendableOutputs { .. } => {},
+                       Event::ChannelClosed { .. } => {},
                        _ => panic!("Unexpected event: {:?}", event),
                }
 
index 2a8ff0c6a1807f91d07f71a64969343328812c81..5e0778cf7c7b87f58c0de61462ded34342e355b6 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-block-sync"
-version = "0.0.100"
+version = "0.0.101"
 authors = ["Jeffrey Czyz", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "http://github.com/rust-bitcoin/rust-lightning"
@@ -15,12 +15,11 @@ rpc-client = [ "serde", "serde_json", "chunked_transfer" ]
 
 [dependencies]
 bitcoin = "0.27"
-lightning = { version = "0.0.100", path = "../lightning" }
+lightning = { version = "0.0.101", path = "../lightning" }
 tokio = { version = "1.0", features = [ "io-util", "net", "time" ], optional = true }
 serde = { version = "1.0", features = ["derive"], optional = true }
 serde_json = { version = "1.0", optional = true }
 chunked_transfer = { version = "1.4", optional = true }
-futures = { version = "0.3" }
 
 [dev-dependencies]
 tokio = { version = "1.0", features = [ "macros", "rt" ] }
index e8e1427bdb654ffa432d069faf37457fc5a74cea..358076f4c25a883f5e571b421f9d3ff1ebe20f27 100644 (file)
@@ -1,11 +1,12 @@
-use crate::{BlockHeaderData, BlockSourceError};
 use crate::http::{BinaryResponse, JsonResponse};
 use crate::utils::hex_to_uint256;
+use crate::{BlockHeaderData, BlockSourceError};
 
 use bitcoin::blockdata::block::{Block, BlockHeader};
 use bitcoin::consensus::encode;
 use bitcoin::hash_types::{BlockHash, TxMerkleNode, Txid};
-use bitcoin::hashes::hex::{ToHex, FromHex};
+use bitcoin::hashes::hex::{FromHex, ToHex};
+use bitcoin::Transaction;
 
 use serde::Deserialize;
 
@@ -104,7 +105,6 @@ impl TryFrom<GetHeaderResponse> for BlockHeaderData {
        }
 }
 
-
 /// Converts a JSON value into a block. Assumes the block is hex-encoded in a JSON string.
 impl TryInto<Block> for JsonResponse {
        type Error = std::io::Error;
@@ -181,13 +181,77 @@ impl TryInto<Txid> for JsonResponse {
        }
 }
 
+/// Converts a JSON value into a transaction. WATCH OUT! this cannot be used for zero-input transactions
+/// (e.g. createrawtransaction). See https://github.com/rust-bitcoin/rust-bitcoincore-rpc/issues/197
+impl TryInto<Transaction> for JsonResponse {
+       type Error = std::io::Error;
+       fn try_into(self) -> std::io::Result<Transaction> {
+               let hex_tx = if self.0.is_object() {
+                       // result is json encoded
+                       match &self.0["hex"] {
+                               // result has hex field
+                               serde_json::Value::String(hex_data) => match self.0["complete"] {
+                                       // result may or may not be signed (e.g. signrawtransactionwithwallet)
+                                       serde_json::Value::Bool(x) => {
+                                               if x == false {
+                                                       let reason = match &self.0["errors"][0]["error"] {
+                                                               serde_json::Value::String(x) => x.as_str(),
+                                                               _ => "Unknown error",
+                                                       };
+
+                                                       return Err(std::io::Error::new(
+                                                               std::io::ErrorKind::InvalidData,
+                                                               format!("transaction couldn't be signed. {}", reason),
+                                                       ));
+                                               } else {
+                                                       hex_data
+                                               }
+                                       }
+                                       // result is a complete transaction (e.g. getrawtranaction verbose)
+                                       _ => hex_data,
+                               },
+                               _ => return Err(std::io::Error::new(
+                                                       std::io::ErrorKind::InvalidData,
+                                                       "expected JSON string",
+                                       )),
+                       }
+               } else {
+                       // result is plain text (e.g. getrawtransaction no verbose)
+                       match self.0.as_str() {
+                               Some(hex_tx) => hex_tx,
+                               None => {
+                                       return Err(std::io::Error::new(
+                                               std::io::ErrorKind::InvalidData,
+                                               "expected JSON string",
+                                       ))
+                               }
+                       }
+               };
+
+               match Vec::<u8>::from_hex(hex_tx) {
+                       Err(_) => Err(std::io::Error::new(
+                               std::io::ErrorKind::InvalidData,
+                               "invalid hex data",
+                       )),
+                       Ok(tx_data) => match encode::deserialize(&tx_data) {
+                               Err(_) => Err(std::io::Error::new(
+                                       std::io::ErrorKind::InvalidData,
+                                       "invalid transaction",
+                               )),
+                               Ok(tx) => Ok(tx),
+                       },
+               }
+       }
+}
+
 #[cfg(test)]
 pub(crate) mod tests {
        use super::*;
        use bitcoin::blockdata::constants::genesis_block;
-       use bitcoin::consensus::encode;
        use bitcoin::hashes::Hash;
        use bitcoin::network::constants::Network;
+       use serde_json::value::Number;
+       use serde_json::Value;
 
        /// Converts from `BlockHeaderData` into a `GetHeaderResponse` JSON value.
        impl From<BlockHeaderData> for serde_json::Value {
@@ -541,4 +605,101 @@ pub(crate) mod tests {
                        Ok(txid) => assert_eq!(txid, target_txid),
                }
        }
+
+       // TryInto<Transaction> can be used in two ways, first with plain hex response where data is
+       // the hex encoded transaction (e.g. as a result of getrawtransaction) or as a JSON object
+       // where the hex encoded transaction can be found in the hex field of the object (if present)
+       // (e.g. as a result of signrawtransactionwithwallet).
+
+       // plain hex transaction
+
+       #[test]
+       fn into_tx_from_json_response_with_invalid_hex_data() {
+               let response = JsonResponse(serde_json::json!("foobar"));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+                               assert_eq!(e.get_ref().unwrap().to_string(), "invalid hex data");
+                       }
+                       Ok(_) => panic!("Expected error"),
+               }
+       }
+
+       #[test]
+       fn into_tx_from_json_response_with_invalid_data_type() {
+               let response = JsonResponse(Value::Number(Number::from_f64(1.0).unwrap()));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+                               assert_eq!(e.get_ref().unwrap().to_string(), "expected JSON string");
+                       }
+                       Ok(_) => panic!("Expected error"),
+               }
+       }
+
+       #[test]
+       fn into_tx_from_json_response_with_invalid_tx_data() {
+               let response = JsonResponse(serde_json::json!("abcd"));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+                               assert_eq!(e.get_ref().unwrap().to_string(), "invalid transaction");
+                       }
+                       Ok(_) => panic!("Expected error"),
+               }
+       }
+
+       #[test]
+       fn into_tx_from_json_response_with_valid_tx_data_plain() {
+               let genesis_block = genesis_block(Network::Bitcoin);
+               let target_tx = genesis_block.txdata.get(0).unwrap();
+               let response = JsonResponse(serde_json::json!(encode::serialize_hex(&target_tx)));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => panic!("Unexpected error: {:?}", e),
+                       Ok(tx) => assert_eq!(&tx, target_tx),
+               }
+       }
+
+       #[test]
+       fn into_tx_from_json_response_with_valid_tx_data_hex_field() {
+               let genesis_block = genesis_block(Network::Bitcoin);
+               let target_tx = genesis_block.txdata.get(0).unwrap();
+               let response = JsonResponse(serde_json::json!({"hex": encode::serialize_hex(&target_tx)}));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => panic!("Unexpected error: {:?}", e),
+                       Ok(tx) => assert_eq!(&tx, target_tx),
+               }
+       }
+
+       // transaction in hex field of JSON object
+
+       #[test]
+       fn into_tx_from_json_response_with_no_hex_field() {
+               let response = JsonResponse(serde_json::json!({ "error": "foo" }));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+                               assert_eq!(
+                                       e.get_ref().unwrap().to_string(),
+                                       "expected JSON string"
+                               );
+                       }
+                       Ok(_) => panic!("Expected error"),
+               }
+       }
+
+       #[test]
+       fn into_tx_from_json_response_not_signed() {
+               let response = JsonResponse(serde_json::json!({ "hex": "foo", "complete": false }));
+               match TryInto::<Transaction>::try_into(response) {
+                       Err(e) => {
+                               assert_eq!(e.kind(), std::io::ErrorKind::InvalidData);
+                               assert!(
+                                       e.get_ref().unwrap().to_string().contains(
+                                       "transaction couldn't be signed")
+                               );
+                       }
+                       Ok(_) => panic!("Expected error"),
+               }
+       }
 }
index 0721babfde3d1b626051ba1ccccb7240d1f5a5a7..a3935edf5cb6e9880217196112d8f4ead5bacd57 100644 (file)
@@ -27,9 +27,10 @@ const TCP_STREAM_TIMEOUT: Duration = Duration::from_secs(5);
 
 /// Timeout for reading the first byte of a response. This is separate from the general read
 /// timeout as it is not uncommon for Bitcoin Core to be blocked waiting on UTXO cache flushes for
-/// upwards of a minute or more. Note that we always retry once when we time out, so the maximum
-/// time we allow Bitcoin Core to block for is twice this value.
-const TCP_STREAM_RESPONSE_TIMEOUT: Duration = Duration::from_secs(120);
+/// upwards of 10 minutes on slow devices (e.g. RPis with SSDs over USB). Note that we always retry
+/// once when we time out, so the maximum time we allow Bitcoin Core to block for is twice this
+/// value.
+const TCP_STREAM_RESPONSE_TIMEOUT: Duration = Duration::from_secs(300);
 
 /// Maximum HTTP message header size in bytes.
 const MAX_HTTP_MESSAGE_HEADER_SIZE: usize = 8192;
index baa9a79c5d5af3a055fdbc1d7894dccb47338e32..1169d9283cb67ceae0635d9a66e1a6104194d774 100644 (file)
@@ -1,7 +1,7 @@
 [package]
 name = "lightning-invoice"
 description = "Data structures to parse and serialize BOLT11 lightning invoices"
-version = "0.8.0"
+version = "0.9.0"
 authors = ["Sebastian Geisler <sgeisler@wh2.tu-dresden.de>"]
 documentation = "https://docs.rs/lightning-invoice/"
 license = "MIT OR Apache-2.0"
@@ -10,11 +10,11 @@ readme = "README.md"
 
 [dependencies]
 bech32 = "0.8"
-lightning = { version = "0.0.100", path = "../lightning" }
+lightning = { version = "0.0.101", path = "../lightning" }
 secp256k1 = { version = "0.20", features = ["recovery"] }
 num-traits = "0.2.8"
 bitcoin_hashes = "0.10"
 
 [dev-dependencies]
 hex = "0.3"
-lightning = { version = "0.0.100", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.101", path = "../lightning", features = ["_test_utils"] }
index df2bbfd8f12459381d3296dc44499cffa2030d07..409fa803cab9cac20f423e21483d0bf58d8face8 100644 (file)
@@ -115,11 +115,11 @@ mod test {
                let amt_msat = invoice.amount_pico_btc().unwrap() / 10;
                let first_hops = nodes[0].node.list_usable_channels();
                let last_hops = invoice.route_hints();
-               let network_graph = nodes[0].net_graph_msg_handler.network_graph.read().unwrap();
+               let network_graph = &nodes[0].net_graph_msg_handler.network_graph;
                let logger = test_utils::TestLogger::new();
                let route = router::get_route(
                        &nodes[0].node.get_our_node_id(),
-                       &network_graph,
+                       network_graph,
                        &invoice.recover_payee_pub_key(),
                        Some(invoice.features().unwrap().clone()),
                        Some(&first_hops.iter().collect::<Vec<_>>()),
index 053766e10f3f7b164d779739c8231b81c87d8817..ad13c8040f24204745eceae06fa2360fd4ab7f39 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-net-tokio"
-version = "0.0.100"
+version = "0.0.101"
 authors = ["Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rust-bitcoin/rust-lightning/"
@@ -12,7 +12,7 @@ edition = "2018"
 
 [dependencies]
 bitcoin = "0.27"
-lightning = { version = "0.0.100", path = "../lightning" }
+lightning = { version = "0.0.101", path = "../lightning" }
 tokio = { version = "1.0", features = [ "io-util", "macros", "rt", "sync", "net", "time" ] }
 
 [dev-dependencies]
index 25c161d2aeb07f8ba5305956e93c9299aca303f1..13a4c0d934fe264fa5be936d440951c5c5bee1dd 100644 (file)
 //! async fn connect_to_node(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, their_node_id: PublicKey, addr: SocketAddr) {
 //!    lightning_net_tokio::connect_outbound(peer_manager, their_node_id, addr).await;
 //!    loop {
-//!            channel_manager.await_persistable_update();
-//!            channel_manager.process_pending_events(&|event| {
-//!                    // Handle the event!
-//!            });
-//!            chain_monitor.process_pending_events(&|event| {
+//!            let event_handler = |event: &Event| {
 //!                    // Handle the event!
-//!            });
+//!            };
+//!            channel_manager.await_persistable_update();
+//!            channel_manager.process_pending_events(&event_handler);
+//!            chain_monitor.process_pending_events(&event_handler);
 //!    }
 //! }
 //!
 //! async fn accept_socket(peer_manager: PeerManager, chain_monitor: Arc<ChainMonitor>, channel_manager: ChannelManager, socket: TcpStream) {
 //!    lightning_net_tokio::setup_inbound(peer_manager, socket);
 //!    loop {
-//!            channel_manager.await_persistable_update();
-//!            channel_manager.process_pending_events(&|event| {
-//!                    // Handle the event!
-//!            });
-//!            chain_monitor.process_pending_events(&|event| {
+//!            let event_handler = |event: &Event| {
 //!                    // Handle the event!
-//!            });
+//!            };
+//!            channel_manager.await_persistable_update();
+//!            channel_manager.process_pending_events(&event_handler);
+//!            chain_monitor.process_pending_events(&event_handler);
 //!    }
 //! }
 //! ```
@@ -494,7 +492,6 @@ mod tests {
                fn handle_node_announcement(&self, _msg: &NodeAnnouncement) -> Result<bool, LightningError> { Ok(false) }
                fn handle_channel_announcement(&self, _msg: &ChannelAnnouncement) -> Result<bool, LightningError> { Ok(false) }
                fn handle_channel_update(&self, _msg: &ChannelUpdate) -> Result<bool, LightningError> { Ok(false) }
-               fn handle_htlc_fail_channel_update(&self, _update: &HTLCFailChannelUpdate) { }
                fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)> { Vec::new() }
                fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec<NodeAnnouncement> { Vec::new() }
                fn sync_routing_table(&self, _their_node_id: &PublicKey, _init_msg: &Init) { }
index cc585b9c0e8c0df612120c63322ca57c66ca094e..ded73551e75c4254c9ef646eb075d488e8c5a9ae 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning-persister"
-version = "0.0.100"
+version = "0.0.101"
 authors = ["Valentine Wallace", "Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rust-bitcoin/rust-lightning/"
@@ -13,11 +13,11 @@ unstable = ["lightning/unstable"]
 
 [dependencies]
 bitcoin = "0.27"
-lightning = { version = "0.0.100", path = "../lightning" }
+lightning = { version = "0.0.101", path = "../lightning" }
 libc = "0.2"
 
 [target.'cfg(windows)'.dependencies]
 winapi = { version = "0.3", features = ["winbase"] }
 
 [dev-dependencies]
-lightning = { version = "0.0.100", path = "../lightning", features = ["_test_utils"] }
+lightning = { version = "0.0.101", path = "../lightning", features = ["_test_utils"] }
index d68a06d71b4fe58ec1af2bcb1117df8f1dddd7e5..6cfa540cf1ec9c32a89a13bf1ba995e8dd3808f5 100644 (file)
@@ -182,11 +182,11 @@ mod tests {
        use bitcoin::Txid;
        use lightning::chain::channelmonitor::{Persist, ChannelMonitorUpdateErr};
        use lightning::chain::transaction::OutPoint;
-       use lightning::{check_closed_broadcast, check_added_monitors};
+       use lightning::{check_closed_broadcast, check_closed_event, check_added_monitors};
        use lightning::ln::features::InitFeatures;
        use lightning::ln::functional_test_utils::*;
        use lightning::ln::msgs::ErrorAction;
-       use lightning::util::events::{MessageSendEventsProvider, MessageSendEvent};
+       use lightning::util::events::{ClosureReason, Event, MessageSendEventsProvider, MessageSendEvent};
        use lightning::util::test_utils;
        use std::fs;
        #[cfg(target_os = "windows")]
@@ -259,6 +259,7 @@ mod tests {
                // Force close because cooperative close doesn't result in any persisted
                // updates.
                nodes[0].node.force_close_channel(&nodes[0].node.list_channels()[0].channel_id).unwrap();
+               check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
                check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
 
@@ -268,6 +269,7 @@ mod tests {
                let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[0].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
                connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[0].clone()]});
                check_closed_broadcast!(nodes[1], true);
+               check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                check_added_monitors!(nodes[1], 1);
 
                // Make sure everything is persisted as expected after close.
@@ -291,6 +293,7 @@ mod tests {
                let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
                let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
                nodes[1].node.force_close_channel(&chan.2).unwrap();
+               check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed);
                let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
 
                // Set the persister's directory to read-only, which should result in
@@ -327,6 +330,7 @@ mod tests {
                let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
                let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
                nodes[1].node.force_close_channel(&chan.2).unwrap();
+               check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed);
                let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
 
                // Create the persister with an invalid directory name and test that the
index 1ef4c74f991108778aabe77d6fbc633bd1314ba0..9d6217b04746f81c4ecd38fa9d5090b40feb4aff 100644 (file)
@@ -1,6 +1,6 @@
 [package]
 name = "lightning"
-version = "0.0.100"
+version = "0.0.101"
 authors = ["Matt Corallo"]
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/rust-bitcoin/rust-lightning/"
index 8969427a0f960bc5766c153c8479a46974035b93..0d3f87645ce4c4c1250d2d42a3a085f419ac13a6 100644 (file)
@@ -30,12 +30,13 @@ use chain;
 use chain::{Filter, WatchedOutput};
 use chain::chaininterface::{BroadcasterInterface, FeeEstimator};
 use chain::channelmonitor;
-use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateErr, MonitorEvent, Persist, TransactionOutputs};
+use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateErr, Balance, MonitorEvent, Persist, TransactionOutputs};
 use chain::transaction::{OutPoint, TransactionData};
 use chain::keysinterface::Sign;
 use util::logger::Logger;
 use util::events;
 use util::events::EventHandler;
+use ln::channelmanager::ChannelDetails;
 
 use prelude::*;
 use sync::RwLock;
@@ -140,11 +141,36 @@ where C::Target: chain::Filter,
                }
        }
 
+       /// Gets the balances in the contained [`ChannelMonitor`]s which are claimable on-chain or
+       /// claims which are awaiting confirmation.
+       ///
+       /// Includes the balances from each [`ChannelMonitor`] *except* those included in
+       /// `ignored_channels`, allowing you to filter out balances from channels which are still open
+       /// (and whose balance should likely be pulled from the [`ChannelDetails`]).
+       ///
+       /// See [`ChannelMonitor::get_claimable_balances`] for more details on the exact criteria for
+       /// inclusion in the return value.
+       pub fn get_claimable_balances(&self, ignored_channels: &[&ChannelDetails]) -> Vec<Balance> {
+               let mut ret = Vec::new();
+               let monitors = self.monitors.read().unwrap();
+               for (_, monitor) in monitors.iter().filter(|(funding_outpoint, _)| {
+                       for chan in ignored_channels {
+                               if chan.funding_txo.as_ref() == Some(funding_outpoint) {
+                                       return false;
+                               }
+                       }
+                       true
+               }) {
+                       ret.append(&mut monitor.get_claimable_balances());
+               }
+               ret
+       }
+
        #[cfg(any(test, feature = "fuzztarget", feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                use util::events::EventsProvider;
                let events = core::cell::RefCell::new(Vec::new());
-               let event_handler = |event| events.borrow_mut().push(event);
+               let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
@@ -332,7 +358,7 @@ impl<ChannelSigner: Sign, C: Deref, T: Deref, F: Deref, L: Deref, P: Deref> even
                        pending_events.append(&mut monitor.get_and_clear_pending_events());
                }
                for event in pending_events.drain(..) {
-                       handler.handle_event(event);
+                       handler.handle_event(&event);
                }
        }
 }
index dc009c4d52cf3e5ef228e505abda9bf3b25d5ea2..b89c58646fbfd960ccf98a5736fc4d300b26db8f 100644 (file)
@@ -190,8 +190,8 @@ pub enum MonitorEvent {
        /// A monitor event containing an HTLCUpdate.
        HTLCEvent(HTLCUpdate),
 
-       /// A monitor event that the Channel's commitment transaction was broadcasted.
-       CommitmentTxBroadcasted(OutPoint),
+       /// A monitor event that the Channel's commitment transaction was confirmed.
+       CommitmentTxConfirmed(OutPoint),
 }
 
 /// Simple structure sent back by `chain::Watch` when an HTLC from a forward channel is detected on
@@ -232,8 +232,13 @@ pub(crate) const CLTV_CLAIM_BUFFER: u32 = 18;
 /// with at worst this delay, so we are not only using this value as a mercy for them but also
 /// us as a safeguard to delay with enough time.
 pub(crate) const LATENCY_GRACE_PERIOD_BLOCKS: u32 = 3;
-/// Number of blocks we wait on seeing a HTLC output being solved before we fail corresponding inbound
-/// HTLCs. This prevents us from failing backwards and then getting a reorg resulting in us losing money.
+/// Number of blocks we wait on seeing a HTLC output being solved before we fail corresponding
+/// inbound HTLCs. This prevents us from failing backwards and then getting a reorg resulting in us
+/// losing money.
+///
+/// Note that this is a library-wide security assumption. If a reorg deeper than this number of
+/// blocks occurs, counterparties may be able to steal funds or claims made by and balances exposed
+/// by a  [`ChannelMonitor`] may be incorrect.
 // We also use this delay to be sure we can remove our in-flight claim txn from bump candidates buffer.
 // It may cause spurious generation of bumped claim txn but that's alright given the outpoint is already
 // solved by a previous claim tx. What we want to avoid is reorg evicting our claim tx and us not
@@ -272,11 +277,15 @@ struct HolderSignedTx {
        b_htlc_key: PublicKey,
        delayed_payment_key: PublicKey,
        per_commitment_point: PublicKey,
-       feerate_per_kw: u32,
        htlc_outputs: Vec<(HTLCOutputInCommitment, Option<Signature>, Option<HTLCSource>)>,
+       to_self_value_sat: u64,
+       feerate_per_kw: u32,
 }
 impl_writeable_tlv_based!(HolderSignedTx, {
        (0, txid, required),
+       // Note that this is filled in with data from OnchainTxHandler if it's missing.
+       // For HolderSignedTx objects serialized with 0.0.100+, this should be filled in.
+       (1, to_self_value_sat, (default_value, u64::max_value())),
        (2, revocation_key, required),
        (4, a_htlc_key, required),
        (6, b_htlc_key, required),
@@ -286,26 +295,18 @@ impl_writeable_tlv_based!(HolderSignedTx, {
        (14, htlc_outputs, vec_type)
 });
 
-/// We use this to track counterparty commitment transactions and htlcs outputs and
-/// use it to generate any justice or 2nd-stage preimage/timeout transactions.
+/// We use this to track static counterparty commitment transaction data and to generate any
+/// justice or 2nd-stage preimage/timeout transactions.
 #[derive(PartialEq)]
-struct CounterpartyCommitmentTransaction {
+struct CounterpartyCommitmentParameters {
        counterparty_delayed_payment_base_key: PublicKey,
        counterparty_htlc_base_key: PublicKey,
        on_counterparty_tx_csv: u16,
-       per_htlc: HashMap<Txid, Vec<HTLCOutputInCommitment>>
 }
 
-impl Writeable for CounterpartyCommitmentTransaction {
+impl Writeable for CounterpartyCommitmentParameters {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.write_all(&byte_utils::be64_to_array(self.per_htlc.len() as u64))?;
-               for (ref txid, ref htlcs) in self.per_htlc.iter() {
-                       w.write_all(&txid[..])?;
-                       w.write_all(&byte_utils::be64_to_array(htlcs.len() as u64))?;
-                       for &ref htlc in htlcs.iter() {
-                               htlc.write(w)?;
-                       }
-               }
+               w.write_all(&byte_utils::be64_to_array(0))?;
                write_tlv_fields!(w, {
                        (0, self.counterparty_delayed_payment_base_key, required),
                        (2, self.counterparty_htlc_base_key, required),
@@ -314,23 +315,20 @@ impl Writeable for CounterpartyCommitmentTransaction {
                Ok(())
        }
 }
-impl Readable for CounterpartyCommitmentTransaction {
+impl Readable for CounterpartyCommitmentParameters {
        fn read<R: io::Read>(r: &mut R) -> Result<Self, DecodeError> {
                let counterparty_commitment_transaction = {
+                       // Versions prior to 0.0.100 had some per-HTLC state stored here, which is no longer
+                       // used. Read it for compatibility.
                        let per_htlc_len: u64 = Readable::read(r)?;
-                       let mut per_htlc = HashMap::with_capacity(cmp::min(per_htlc_len as usize, MAX_ALLOC_SIZE / 64));
                        for _  in 0..per_htlc_len {
-                               let txid: Txid = Readable::read(r)?;
+                               let _txid: Txid = Readable::read(r)?;
                                let htlcs_count: u64 = Readable::read(r)?;
-                               let mut htlcs = Vec::with_capacity(cmp::min(htlcs_count as usize, MAX_ALLOC_SIZE / 32));
                                for _ in 0..htlcs_count {
-                                       let htlc = Readable::read(r)?;
-                                       htlcs.push(htlc);
-                               }
-                               if let Some(_) = per_htlc.insert(txid, htlcs) {
-                                       return Err(DecodeError::InvalidValue);
+                                       let _htlc: HTLCOutputInCommitment = Readable::read(r)?;
                                }
                        }
+
                        let mut counterparty_delayed_payment_base_key = OptionDeserWrapper(None);
                        let mut counterparty_htlc_base_key = OptionDeserWrapper(None);
                        let mut on_counterparty_tx_csv: u16 = 0;
@@ -339,11 +337,10 @@ impl Readable for CounterpartyCommitmentTransaction {
                                (2, counterparty_htlc_base_key, required),
                                (4, on_counterparty_tx_csv, required),
                        });
-                       CounterpartyCommitmentTransaction {
+                       CounterpartyCommitmentParameters {
                                counterparty_delayed_payment_base_key: counterparty_delayed_payment_base_key.0.unwrap(),
                                counterparty_htlc_base_key: counterparty_htlc_base_key.0.unwrap(),
                                on_counterparty_tx_csv,
-                               per_htlc,
                        }
                };
                Ok(counterparty_commitment_transaction)
@@ -364,12 +361,21 @@ struct OnchainEventEntry {
 impl OnchainEventEntry {
        fn confirmation_threshold(&self) -> u32 {
                let mut conf_threshold = self.height + ANTI_REORG_DELAY - 1;
-               if let OnchainEvent::MaturingOutput {
-                       descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(ref descriptor)
-               } = self.event {
-                       // A CSV'd transaction is confirmable in block (input height) + CSV delay, which means
-                       // it's broadcastable when we see the previous block.
-                       conf_threshold = cmp::max(conf_threshold, self.height + descriptor.to_self_delay as u32 - 1);
+               match self.event {
+                       OnchainEvent::MaturingOutput {
+                               descriptor: SpendableOutputDescriptor::DelayedPaymentOutput(ref descriptor)
+                       } => {
+                               // A CSV'd transaction is confirmable in block (input height) + CSV delay, which means
+                               // it's broadcastable when we see the previous block.
+                               conf_threshold = cmp::max(conf_threshold, self.height + descriptor.to_self_delay as u32 - 1);
+                       },
+                       OnchainEvent::FundingSpendConfirmation { on_local_output_csv: Some(csv), .. } |
+                       OnchainEvent::HTLCSpendConfirmation { on_to_local_output_csv: Some(csv), .. } => {
+                               // A CSV'd transaction is confirmable in block (input height) + CSV delay, which means
+                               // it's broadcastable when we see the previous block.
+                               conf_threshold = cmp::max(conf_threshold, self.height + csv as u32 - 1);
+                       },
+                       _ => {},
                }
                conf_threshold
        }
@@ -383,17 +389,47 @@ impl OnchainEventEntry {
 /// once they mature to enough confirmations (ANTI_REORG_DELAY)
 #[derive(PartialEq)]
 enum OnchainEvent {
-       /// HTLC output getting solved by a timeout, at maturation we pass upstream payment source information to solve
-       /// inbound HTLC in backward channel. Note, in case of preimage, we pass info to upstream without delay as we can
-       /// only win from it, so it's never an OnchainEvent
+       /// An outbound HTLC failing after a transaction is confirmed. Used
+       ///  * when an outbound HTLC output is spent by us after the HTLC timed out
+       ///  * an outbound HTLC which was not present in the commitment transaction which appeared
+       ///    on-chain (either because it was not fully committed to or it was dust).
+       /// Note that this is *not* used for preimage claims, as those are passed upstream immediately,
+       /// appearing only as an `HTLCSpendConfirmation`, below.
        HTLCUpdate {
                source: HTLCSource,
                payment_hash: PaymentHash,
                onchain_value_satoshis: Option<u64>,
+               /// None in the second case, above, ie when there is no relevant output in the commitment
+               /// transaction which appeared on chain.
+               input_idx: Option<u32>,
        },
        MaturingOutput {
                descriptor: SpendableOutputDescriptor,
        },
+       /// A spend of the funding output, either a commitment transaction or a cooperative closing
+       /// transaction.
+       FundingSpendConfirmation {
+               /// The CSV delay for the output of the funding spend transaction (implying it is a local
+               /// commitment transaction, and this is the delay on the to_self output).
+               on_local_output_csv: Option<u16>,
+       },
+       /// A spend of a commitment transaction HTLC output, set in the cases where *no* `HTLCUpdate`
+       /// is constructed. This is used when
+       ///  * an outbound HTLC is claimed by our counterparty with a preimage, causing us to
+       ///    immediately claim the HTLC on the inbound edge and track the resolution here,
+       ///  * an inbound HTLC is claimed by our counterparty (with a timeout),
+       ///  * an inbound HTLC is claimed by us (with a preimage).
+       ///  * a revoked-state HTLC transaction was broadcasted, which was claimed by the revocation
+       ///    signature.
+       HTLCSpendConfirmation {
+               input_idx: u32,
+               /// If the claim was made by either party with a preimage, this is filled in
+               preimage: Option<PaymentPreimage>,
+               /// If the claim was made by us on an inbound HTLC against a local commitment transaction,
+               /// we set this to the output CSV value which we will have to wait until to spend the
+               /// output (and generate a SpendableOutput event).
+               on_to_local_output_csv: Option<u16>,
+       },
 }
 
 impl Writeable for OnchainEventEntry {
@@ -430,10 +466,20 @@ impl_writeable_tlv_based_enum_upgradable!(OnchainEvent,
                (0, source, required),
                (1, onchain_value_satoshis, option),
                (2, payment_hash, required),
+               (3, input_idx, option),
        },
        (1, MaturingOutput) => {
                (0, descriptor, required),
        },
+       (3, FundingSpendConfirmation) => {
+               (0, on_local_output_csv, option),
+       },
+       (5, HTLCSpendConfirmation) => {
+               (0, input_idx, required),
+               (2, preimage, option),
+               (4, on_to_local_output_csv, option),
+       },
+
 );
 
 #[cfg_attr(any(test, feature = "fuzztarget", feature = "_test_utils"), derive(PartialEq))]
@@ -494,6 +540,72 @@ impl_writeable_tlv_based_enum_upgradable!(ChannelMonitorUpdateStep,
        },
 );
 
+/// Details about the balance(s) available for spending once the channel appears on chain.
+///
+/// See [`ChannelMonitor::get_claimable_balances`] for more details on when these will or will not
+/// be provided.
+#[derive(Clone, Debug, PartialEq, Eq)]
+#[cfg_attr(test, derive(PartialOrd, Ord))]
+pub enum Balance {
+       /// The channel is not yet closed (or the commitment or closing transaction has not yet
+       /// appeared in a block). The given balance is claimable (less on-chain fees) if the channel is
+       /// force-closed now.
+       ClaimableOnChannelClose {
+               /// The amount available to claim, in satoshis, excluding the on-chain fees which will be
+               /// required to do so.
+               claimable_amount_satoshis: u64,
+       },
+       /// The channel has been closed, and the given balance is ours but awaiting confirmations until
+       /// we consider it spendable.
+       ClaimableAwaitingConfirmations {
+               /// The amount available to claim, in satoshis, possibly excluding the on-chain fees which
+               /// were spent in broadcasting the transaction.
+               claimable_amount_satoshis: u64,
+               /// The height at which an [`Event::SpendableOutputs`] event will be generated for this
+               /// amount.
+               confirmation_height: u32,
+       },
+       /// The channel has been closed, and the given balance should be ours but awaiting spending
+       /// transaction confirmation. If the spending transaction does not confirm in time, it is
+       /// possible our counterparty can take the funds by broadcasting an HTLC timeout on-chain.
+       ///
+       /// Once the spending transaction confirms, before it has reached enough confirmations to be
+       /// considered safe from chain reorganizations, the balance will instead be provided via
+       /// [`Balance::ClaimableAwaitingConfirmations`].
+       ContentiousClaimable {
+               /// The amount available to claim, in satoshis, excluding the on-chain fees which will be
+               /// required to do so.
+               claimable_amount_satoshis: u64,
+               /// The height at which the counterparty may be able to claim the balance if we have not
+               /// done so.
+               timeout_height: u32,
+       },
+       /// HTLCs which we sent to our counterparty which are claimable after a timeout (less on-chain
+       /// fees) if the counterparty does not know the preimage for the HTLCs. These are somewhat
+       /// likely to be claimed by our counterparty before we do.
+       MaybeClaimableHTLCAwaitingTimeout {
+               /// The amount available to claim, in satoshis, excluding the on-chain fees which will be
+               /// required to do so.
+               claimable_amount_satoshis: u64,
+               /// The height at which we will be able to claim the balance if our counterparty has not
+               /// done so.
+               claimable_height: u32,
+       },
+}
+
+/// An HTLC which has been irrevocably resolved on-chain, and has reached ANTI_REORG_DELAY.
+#[derive(PartialEq)]
+struct IrrevocablyResolvedHTLC {
+       input_idx: u32,
+       /// Only set if the HTLC claim was ours using a payment preimage
+       payment_preimage: Option<PaymentPreimage>,
+}
+
+impl_writeable_tlv_based!(IrrevocablyResolvedHTLC, {
+       (0, input_idx, required),
+       (2, payment_preimage, option),
+});
+
 /// A ChannelMonitor handles chain events (blocks connected and disconnected) and generates
 /// on-chain transactions to ensure no loss of funds occurs.
 ///
@@ -532,7 +644,7 @@ pub(crate) struct ChannelMonitorImpl<Signer: Sign> {
        current_counterparty_commitment_txid: Option<Txid>,
        prev_counterparty_commitment_txid: Option<Txid>,
 
-       counterparty_tx_cache: CounterpartyCommitmentTransaction,
+       counterparty_commitment_params: CounterpartyCommitmentParameters,
        funding_redeemscript: Script,
        channel_value_satoshis: u64,
        // first is the idx of the first of the two revocation points
@@ -606,6 +718,12 @@ pub(crate) struct ChannelMonitorImpl<Signer: Sign> {
        // remote monitor out-of-order with regards to the block view.
        holder_tx_signed: bool,
 
+       funding_spend_confirmed: Option<Txid>,
+       /// The set of HTLCs which have been either claimed or failed on chain and have reached
+       /// the requisite confirmations on the claim/fail transaction (either ANTI_REORG_DELAY or the
+       /// spending CSV for revocable outputs).
+       htlcs_resolved_on_chain: Vec<IrrevocablyResolvedHTLC>,
+
        // We simply modify best_block in Channel's block_connected so that serialization is
        // consistent but hopefully the users' copy handles block_connected in a consistent way.
        // (we do *not*, however, update them in update_monitor to ensure any local user copies keep
@@ -645,7 +763,7 @@ impl<Signer: Sign> PartialEq for ChannelMonitorImpl<Signer> {
                        self.funding_info != other.funding_info ||
                        self.current_counterparty_commitment_txid != other.current_counterparty_commitment_txid ||
                        self.prev_counterparty_commitment_txid != other.prev_counterparty_commitment_txid ||
-                       self.counterparty_tx_cache != other.counterparty_tx_cache ||
+                       self.counterparty_commitment_params != other.counterparty_commitment_params ||
                        self.funding_redeemscript != other.funding_redeemscript ||
                        self.channel_value_satoshis != other.channel_value_satoshis ||
                        self.their_cur_revocation_points != other.their_cur_revocation_points ||
@@ -664,7 +782,9 @@ impl<Signer: Sign> PartialEq for ChannelMonitorImpl<Signer> {
                        self.onchain_events_awaiting_threshold_conf != other.onchain_events_awaiting_threshold_conf ||
                        self.outputs_to_watch != other.outputs_to_watch ||
                        self.lockdown_from_offchain != other.lockdown_from_offchain ||
-                       self.holder_tx_signed != other.holder_tx_signed
+                       self.holder_tx_signed != other.holder_tx_signed ||
+                       self.funding_spend_confirmed != other.funding_spend_confirmed ||
+                       self.htlcs_resolved_on_chain != other.htlcs_resolved_on_chain
                {
                        false
                } else {
@@ -716,7 +836,7 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                self.current_counterparty_commitment_txid.write(writer)?;
                self.prev_counterparty_commitment_txid.write(writer)?;
 
-               self.counterparty_tx_cache.write(writer)?;
+               self.counterparty_commitment_params.write(writer)?;
                self.funding_redeemscript.write(writer)?;
                self.channel_value_satoshis.write(writer)?;
 
@@ -798,7 +918,7 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                                        0u8.write(writer)?;
                                        upd.write(writer)?;
                                },
-                               MonitorEvent::CommitmentTxBroadcasted(_) => 1u8.write(writer)?
+                               MonitorEvent::CommitmentTxConfirmed(_) => 1u8.write(writer)?
                        }
                }
 
@@ -829,7 +949,10 @@ impl<Signer: Sign> Writeable for ChannelMonitorImpl<Signer> {
                self.lockdown_from_offchain.write(writer)?;
                self.holder_tx_signed.write(writer)?;
 
-               write_tlv_fields!(writer, {});
+               write_tlv_fields!(writer, {
+                       (1, self.funding_spend_confirmed, option),
+                       (3, self.htlcs_resolved_on_chain, vec_type),
+               });
 
                Ok(())
        }
@@ -851,7 +974,7 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                let counterparty_channel_parameters = channel_parameters.counterparty_parameters.as_ref().unwrap();
                let counterparty_delayed_payment_base_key = counterparty_channel_parameters.pubkeys.delayed_payment_basepoint;
                let counterparty_htlc_base_key = counterparty_channel_parameters.pubkeys.htlc_basepoint;
-               let counterparty_tx_cache = CounterpartyCommitmentTransaction { counterparty_delayed_payment_base_key, counterparty_htlc_base_key, on_counterparty_tx_csv, per_htlc: HashMap::new() };
+               let counterparty_commitment_params = CounterpartyCommitmentParameters { counterparty_delayed_payment_base_key, counterparty_htlc_base_key, on_counterparty_tx_csv };
 
                let channel_keys_id = keys.channel_keys_id();
                let holder_revocation_basepoint = keys.pubkeys().revocation_basepoint;
@@ -869,8 +992,9 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                                b_htlc_key: tx_keys.countersignatory_htlc_key,
                                delayed_payment_key: tx_keys.broadcaster_delayed_payment_key,
                                per_commitment_point: tx_keys.per_commitment_point,
-                               feerate_per_kw: trusted_tx.feerate_per_kw(),
                                htlc_outputs: Vec::new(), // There are never any HTLCs in the initial commitment transactions
+                               to_self_value_sat: initial_holder_commitment_tx.to_broadcaster_value_sat(),
+                               feerate_per_kw: trusted_tx.feerate_per_kw(),
                        };
                        (holder_commitment_tx, trusted_tx.commitment_number())
                };
@@ -898,7 +1022,7 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
                                current_counterparty_commitment_txid: None,
                                prev_counterparty_commitment_txid: None,
 
-                               counterparty_tx_cache,
+                               counterparty_commitment_params,
                                funding_redeemscript,
                                channel_value_satoshis,
                                their_cur_revocation_points: None,
@@ -926,6 +1050,8 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
 
                                lockdown_from_offchain: false,
                                holder_tx_signed: false,
+                               funding_spend_confirmed: None,
+                               htlcs_resolved_on_chain: Vec::new(),
 
                                best_block,
 
@@ -1234,6 +1360,173 @@ impl<Signer: Sign> ChannelMonitor<Signer> {
        pub fn current_best_block(&self) -> BestBlock {
                self.inner.lock().unwrap().best_block.clone()
        }
+
+       /// Gets the balances in this channel which are either claimable by us if we were to
+       /// force-close the channel now or which are claimable on-chain (possibly awaiting
+       /// confirmation).
+       ///
+       /// Any balances in the channel which are available on-chain (excluding on-chain fees) are
+       /// included here until an [`Event::SpendableOutputs`] event has been generated for the
+       /// balance, or until our counterparty has claimed the balance and accrued several
+       /// confirmations on the claim transaction.
+       ///
+       /// Note that the balances available when you or your counterparty have broadcasted revoked
+       /// state(s) may not be fully captured here.
+       // TODO, fix that ^
+       ///
+       /// See [`Balance`] for additional details on the types of claimable balances which
+       /// may be returned here and their meanings.
+       pub fn get_claimable_balances(&self) -> Vec<Balance> {
+               let mut res = Vec::new();
+               let us = self.inner.lock().unwrap();
+
+               let mut confirmed_txid = us.funding_spend_confirmed;
+               let mut pending_commitment_tx_conf_thresh = None;
+               let funding_spend_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                       if let OnchainEvent::FundingSpendConfirmation { .. } = event.event {
+                               Some((event.txid, event.confirmation_threshold()))
+                       } else { None }
+               });
+               if let Some((txid, conf_thresh)) = funding_spend_pending {
+                       debug_assert!(us.funding_spend_confirmed.is_none(),
+                               "We have a pending funding spend awaiting anti-reorg confirmation, we can't have confirmed it already!");
+                       confirmed_txid = Some(txid);
+                       pending_commitment_tx_conf_thresh = Some(conf_thresh);
+               }
+
+               macro_rules! walk_htlcs {
+                       ($holder_commitment: expr, $htlc_iter: expr) => {
+                               for htlc in $htlc_iter {
+                                       if let Some(htlc_input_idx) = htlc.transaction_output_index {
+                                               if us.htlcs_resolved_on_chain.iter().any(|v| v.input_idx == htlc_input_idx) {
+                                                       assert!(us.funding_spend_confirmed.is_some());
+                                               } else if htlc.offered == $holder_commitment {
+                                                       // If the payment was outbound, check if there's an HTLCUpdate
+                                                       // indicating we have spent this HTLC with a timeout, claiming it back
+                                                       // and awaiting confirmations on it.
+                                                       let htlc_update_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                                                               if let OnchainEvent::HTLCUpdate { input_idx: Some(input_idx), .. } = event.event {
+                                                                       if input_idx == htlc_input_idx { Some(event.confirmation_threshold()) } else { None }
+                                                               } else { None }
+                                                       });
+                                                       if let Some(conf_thresh) = htlc_update_pending {
+                                                               res.push(Balance::ClaimableAwaitingConfirmations {
+                                                                       claimable_amount_satoshis: htlc.amount_msat / 1000,
+                                                                       confirmation_height: conf_thresh,
+                                                               });
+                                                       } else {
+                                                               res.push(Balance::MaybeClaimableHTLCAwaitingTimeout {
+                                                                       claimable_amount_satoshis: htlc.amount_msat / 1000,
+                                                                       claimable_height: htlc.cltv_expiry,
+                                                               });
+                                                       }
+                                               } else if us.payment_preimages.get(&htlc.payment_hash).is_some() {
+                                                       // Otherwise (the payment was inbound), only expose it as claimable if
+                                                       // we know the preimage.
+                                                       // Note that if there is a pending claim, but it did not use the
+                                                       // preimage, we lost funds to our counterparty! We will then continue
+                                                       // to show it as ContentiousClaimable until ANTI_REORG_DELAY.
+                                                       let htlc_spend_pending = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                                                               if let OnchainEvent::HTLCSpendConfirmation { input_idx, preimage, .. } = event.event {
+                                                                       if input_idx == htlc_input_idx {
+                                                                               Some((event.confirmation_threshold(), preimage.is_some()))
+                                                                       } else { None }
+                                                               } else { None }
+                                                       });
+                                                       if let Some((conf_thresh, true)) = htlc_spend_pending {
+                                                               res.push(Balance::ClaimableAwaitingConfirmations {
+                                                                       claimable_amount_satoshis: htlc.amount_msat / 1000,
+                                                                       confirmation_height: conf_thresh,
+                                                               });
+                                                       } else {
+                                                               res.push(Balance::ContentiousClaimable {
+                                                                       claimable_amount_satoshis: htlc.amount_msat / 1000,
+                                                                       timeout_height: htlc.cltv_expiry,
+                                                               });
+                                                       }
+                                               }
+                                       }
+                               }
+                       }
+               }
+
+               if let Some(txid) = confirmed_txid {
+                       let mut found_commitment_tx = false;
+                       if Some(txid) == us.current_counterparty_commitment_txid || Some(txid) == us.prev_counterparty_commitment_txid {
+                               walk_htlcs!(false, us.counterparty_claimable_outpoints.get(&txid).unwrap().iter().map(|(a, _)| a));
+                               if let Some(conf_thresh) = pending_commitment_tx_conf_thresh {
+                                       if let Some(value) = us.onchain_events_awaiting_threshold_conf.iter().find_map(|event| {
+                                               if let OnchainEvent::MaturingOutput {
+                                                       descriptor: SpendableOutputDescriptor::StaticPaymentOutput(descriptor)
+                                               } = &event.event {
+                                                       Some(descriptor.output.value)
+                                               } else { None }
+                                       }) {
+                                               res.push(Balance::ClaimableAwaitingConfirmations {
+                                                       claimable_amount_satoshis: value,
+                                                       confirmation_height: conf_thresh,
+                                               });
+                                       } else {
+                                               // If a counterparty commitment transaction is awaiting confirmation, we
+                                               // should either have a StaticPaymentOutput MaturingOutput event awaiting
+                                               // confirmation with the same height or have never met our dust amount.
+                                       }
+                               }
+                               found_commitment_tx = true;
+                       } else if txid == us.current_holder_commitment_tx.txid {
+                               walk_htlcs!(true, us.current_holder_commitment_tx.htlc_outputs.iter().map(|(a, _, _)| a));
+                               if let Some(conf_thresh) = pending_commitment_tx_conf_thresh {
+                                       res.push(Balance::ClaimableAwaitingConfirmations {
+                                               claimable_amount_satoshis: us.current_holder_commitment_tx.to_self_value_sat,
+                                               confirmation_height: conf_thresh,
+                                       });
+                               }
+                               found_commitment_tx = true;
+                       } else if let Some(prev_commitment) = &us.prev_holder_signed_commitment_tx {
+                               if txid == prev_commitment.txid {
+                                       walk_htlcs!(true, prev_commitment.htlc_outputs.iter().map(|(a, _, _)| a));
+                                       if let Some(conf_thresh) = pending_commitment_tx_conf_thresh {
+                                               res.push(Balance::ClaimableAwaitingConfirmations {
+                                                       claimable_amount_satoshis: prev_commitment.to_self_value_sat,
+                                                       confirmation_height: conf_thresh,
+                                               });
+                                       }
+                                       found_commitment_tx = true;
+                               }
+                       }
+                       if !found_commitment_tx {
+                               if let Some(conf_thresh) = pending_commitment_tx_conf_thresh {
+                                       // We blindly assume this is a cooperative close transaction here, and that
+                                       // neither us nor our counterparty misbehaved. At worst we've under-estimated
+                                       // the amount we can claim as we'll punish a misbehaving counterparty.
+                                       res.push(Balance::ClaimableAwaitingConfirmations {
+                                               claimable_amount_satoshis: us.current_holder_commitment_tx.to_self_value_sat,
+                                               confirmation_height: conf_thresh,
+                                       });
+                               }
+                       }
+                       // TODO: Add logic to provide claimable balances for counterparty broadcasting revoked
+                       // outputs.
+               } else {
+                       let mut claimable_inbound_htlc_value_sat = 0;
+                       for (htlc, _, _) in us.current_holder_commitment_tx.htlc_outputs.iter() {
+                               if htlc.transaction_output_index.is_none() { continue; }
+                               if htlc.offered {
+                                       res.push(Balance::MaybeClaimableHTLCAwaitingTimeout {
+                                               claimable_amount_satoshis: htlc.amount_msat / 1000,
+                                               claimable_height: htlc.cltv_expiry,
+                                       });
+                               } else if us.payment_preimages.get(&htlc.payment_hash).is_some() {
+                                       claimable_inbound_htlc_value_sat += htlc.amount_msat / 1000;
+                               }
+                       }
+                       res.push(Balance::ClaimableOnChannelClose {
+                               claimable_amount_satoshis: us.current_holder_commitment_tx.to_self_value_sat + claimable_inbound_htlc_value_sat,
+                       });
+               }
+
+               res
+       }
 }
 
 /// Compares a broadcasted commitment transaction's HTLCs with those in the latest state,
@@ -1292,6 +1585,7 @@ macro_rules! fail_unbroadcast_htlcs {
                                                                        source: (**source).clone(),
                                                                        payment_hash: htlc.payment_hash.clone(),
                                                                        onchain_value_satoshis: Some(htlc.amount_msat / 1000),
+                                                                       input_idx: None,
                                                                },
                                                        };
                                                        log_trace!($logger, "Failing HTLC with payment_hash {} from {} counterparty commitment tx due to broadcast of {} commitment transaction, waiting for confirmation (at height {})",
@@ -1402,7 +1696,6 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                htlcs.push(htlc.0);
                        }
                }
-               self.counterparty_tx_cache.per_htlc.insert(txid, htlcs);
        }
 
        /// Informs this monitor of the latest holder (ie broadcastable) commitment transaction. The
@@ -1424,8 +1717,9 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                b_htlc_key: tx_keys.countersignatory_htlc_key,
                                delayed_payment_key: tx_keys.broadcaster_delayed_payment_key,
                                per_commitment_point: tx_keys.per_commitment_point,
-                               feerate_per_kw: trusted_tx.feerate_per_kw(),
                                htlc_outputs,
+                               to_self_value_sat: holder_commitment_tx.to_broadcaster_value_sat(),
+                               feerate_per_kw: trusted_tx.feerate_per_kw(),
                        }
                };
                self.onchain_tx_handler.provide_latest_holder_tx(holder_commitment_tx);
@@ -1493,7 +1787,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        log_info!(logger, "Broadcasting local {}", log_tx!(tx));
                        broadcaster.broadcast_transaction(tx);
                }
-               self.pending_monitor_events.push(MonitorEvent::CommitmentTxBroadcasted(self.funding_info.0));
+               self.pending_monitor_events.push(MonitorEvent::CommitmentTxConfirmed(self.funding_info.0));
        }
 
        pub fn update_monitor<B: Deref, F: Deref, L: Deref>(&mut self, updates: &ChannelMonitorUpdate, broadcaster: &B, fee_estimator: &F, logger: &L) -> Result<(), MonitorUpdateError>
@@ -1541,7 +1835,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                        } else if !self.holder_tx_signed {
                                                log_error!(logger, "You have a toxic holder commitment transaction avaible in channel monitor, read comment in ChannelMonitor::get_latest_holder_commitment_txn to be informed of manual action to take");
                                        } else {
-                                               // If we generated a MonitorEvent::CommitmentTxBroadcasted, the ChannelManager
+                                               // If we generated a MonitorEvent::CommitmentTxConfirmed, the ChannelManager
                                                // will still give us a ChannelForceClosed event with !should_broadcast, but we
                                                // shouldn't print the scary warning above.
                                                log_info!(logger, "Channel off-chain state closed after we broadcasted our latest commitment transaction.");
@@ -1636,16 +1930,16 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        let per_commitment_key = ignore_error!(SecretKey::from_slice(&secret));
                        let per_commitment_point = PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key);
                        let revocation_pubkey = ignore_error!(chan_utils::derive_public_revocation_key(&self.secp_ctx, &per_commitment_point, &self.holder_revocation_basepoint));
-                       let delayed_key = ignore_error!(chan_utils::derive_public_key(&self.secp_ctx, &PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key), &self.counterparty_tx_cache.counterparty_delayed_payment_base_key));
+                       let delayed_key = ignore_error!(chan_utils::derive_public_key(&self.secp_ctx, &PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key), &self.counterparty_commitment_params.counterparty_delayed_payment_base_key));
 
-                       let revokeable_redeemscript = chan_utils::get_revokeable_redeemscript(&revocation_pubkey, self.counterparty_tx_cache.on_counterparty_tx_csv, &delayed_key);
+                       let revokeable_redeemscript = chan_utils::get_revokeable_redeemscript(&revocation_pubkey, self.counterparty_commitment_params.on_counterparty_tx_csv, &delayed_key);
                        let revokeable_p2wsh = revokeable_redeemscript.to_v0_p2wsh();
 
                        // First, process non-htlc outputs (to_holder & to_counterparty)
                        for (idx, outp) in tx.output.iter().enumerate() {
                                if outp.script_pubkey == revokeable_p2wsh {
-                                       let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_tx_cache.counterparty_delayed_payment_base_key, self.counterparty_tx_cache.counterparty_htlc_base_key, per_commitment_key, outp.value, self.counterparty_tx_cache.on_counterparty_tx_csv);
-                                       let justice_package = PackageTemplate::build_package(commitment_txid, idx as u32, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_tx_cache.on_counterparty_tx_csv as u32, true, height);
+                                       let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, outp.value, self.counterparty_commitment_params.on_counterparty_tx_csv);
+                                       let justice_package = PackageTemplate::build_package(commitment_txid, idx as u32, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_commitment_params.on_counterparty_tx_csv as u32, true, height);
                                        claimable_outpoints.push(justice_package);
                                }
                        }
@@ -1658,7 +1952,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                                tx.output[transaction_output_index as usize].value != htlc.amount_msat / 1000 {
                                                        return (claimable_outpoints, (commitment_txid, watch_outputs)); // Corrupted per_commitment_data, fuck this user
                                                }
-                                               let revk_htlc_outp = RevokedHTLCOutput::build(per_commitment_point, self.counterparty_tx_cache.counterparty_delayed_payment_base_key, self.counterparty_tx_cache.counterparty_htlc_base_key, per_commitment_key, htlc.amount_msat / 1000, htlc.clone());
+                                               let revk_htlc_outp = RevokedHTLCOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, htlc.amount_msat / 1000, htlc.clone());
                                                let justice_package = PackageTemplate::build_package(commitment_txid, transaction_output_index, PackageSolvingData::RevokedHTLCOutput(revk_htlc_outp), htlc.cltv_expiry, true, height);
                                                claimable_outpoints.push(justice_package);
                                        }
@@ -1726,7 +2020,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                        }
                                                        let preimage = if htlc.offered { if let Some(p) = self.payment_preimages.get(&htlc.payment_hash) { Some(*p) } else { None } } else { None };
                                                        if preimage.is_some() || !htlc.offered {
-                                                               let counterparty_htlc_outp = if htlc.offered { PackageSolvingData::CounterpartyOfferedHTLCOutput(CounterpartyOfferedHTLCOutput::build(*revocation_point, self.counterparty_tx_cache.counterparty_delayed_payment_base_key, self.counterparty_tx_cache.counterparty_htlc_base_key, preimage.unwrap(), htlc.clone())) } else { PackageSolvingData::CounterpartyReceivedHTLCOutput(CounterpartyReceivedHTLCOutput::build(*revocation_point, self.counterparty_tx_cache.counterparty_delayed_payment_base_key, self.counterparty_tx_cache.counterparty_htlc_base_key, htlc.clone())) };
+                                                               let counterparty_htlc_outp = if htlc.offered { PackageSolvingData::CounterpartyOfferedHTLCOutput(CounterpartyOfferedHTLCOutput::build(*revocation_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, preimage.unwrap(), htlc.clone())) } else { PackageSolvingData::CounterpartyReceivedHTLCOutput(CounterpartyReceivedHTLCOutput::build(*revocation_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, htlc.clone())) };
                                                                let aggregation = if !htlc.offered { false } else { true };
                                                                let counterparty_package = PackageTemplate::build_package(commitment_txid, transaction_output_index, counterparty_htlc_outp, htlc.cltv_expiry,aggregation, 0);
                                                                claimable_outpoints.push(counterparty_package);
@@ -1760,8 +2054,8 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                let per_commitment_point = PublicKey::from_secret_key(&self.secp_ctx, &per_commitment_key);
 
                log_error!(logger, "Got broadcast of revoked counterparty HTLC transaction, spending {}:{}", htlc_txid, 0);
-               let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_tx_cache.counterparty_delayed_payment_base_key, self.counterparty_tx_cache.counterparty_htlc_base_key, per_commitment_key, tx.output[0].value, self.counterparty_tx_cache.on_counterparty_tx_csv);
-               let justice_package = PackageTemplate::build_package(htlc_txid, 0, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_tx_cache.on_counterparty_tx_csv as u32, true, height);
+               let revk_outp = RevokedOutput::build(per_commitment_point, self.counterparty_commitment_params.counterparty_delayed_payment_base_key, self.counterparty_commitment_params.counterparty_htlc_base_key, per_commitment_key, tx.output[0].value, self.counterparty_commitment_params.on_counterparty_tx_csv);
+               let justice_package = PackageTemplate::build_package(htlc_txid, 0, PackageSolvingData::RevokedOutput(revk_outp), height + self.counterparty_commitment_params.on_counterparty_tx_csv as u32, true, height);
                let claimable_outpoints = vec!(justice_package);
                let outputs = vec![(0, tx.output[0].clone())];
                (claimable_outpoints, Some((htlc_txid, outputs)))
@@ -1811,7 +2105,8 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
        /// Attempts to claim any claimable HTLCs in a commitment transaction which was not (yet)
        /// revoked using data in holder_claimable_outpoints.
        /// Should not be used if check_spend_revoked_transaction succeeds.
-       fn check_spend_holder_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L) -> (Vec<PackageTemplate>, TransactionOutputs) where L::Target: Logger {
+       /// Returns None unless the transaction is definitely one of our commitment transactions.
+       fn check_spend_holder_transaction<L: Deref>(&mut self, tx: &Transaction, height: u32, logger: &L) -> Option<(Vec<PackageTemplate>, TransactionOutputs)> where L::Target: Logger {
                let commitment_txid = tx.txid();
                let mut claim_requests = Vec::new();
                let mut watch_outputs = Vec::new();
@@ -1846,9 +2141,10 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                }
 
                if is_holder_tx {
+                       Some((claim_requests, (commitment_txid, watch_outputs)))
+               } else {
+                       None
                }
-
-               (claim_requests, (commitment_txid, watch_outputs))
        }
 
        pub fn get_latest_holder_commitment_txn<L: Deref>(&mut self, logger: &L) -> Vec<Transaction> where L::Target: Logger {
@@ -1980,20 +2276,32 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                // filters.
                                let prevout = &tx.input[0].previous_output;
                                if prevout.txid == self.funding_info.0.txid && prevout.vout == self.funding_info.0.index as u32 {
+                                       let mut balance_spendable_csv = None;
+                                       log_info!(logger, "Channel closed by funding output spend in txid {}.", log_bytes!(tx.txid()));
                                        if (tx.input[0].sequence >> 8*3) as u8 == 0x80 && (tx.lock_time >> 8*3) as u8 == 0x20 {
                                                let (mut new_outpoints, new_outputs) = self.check_spend_counterparty_transaction(&tx, height, &logger);
                                                if !new_outputs.1.is_empty() {
                                                        watch_outputs.push(new_outputs);
                                                }
+                                               claimable_outpoints.append(&mut new_outpoints);
                                                if new_outpoints.is_empty() {
-                                                       let (mut new_outpoints, new_outputs) = self.check_spend_holder_transaction(&tx, height, &logger);
-                                                       if !new_outputs.1.is_empty() {
-                                                               watch_outputs.push(new_outputs);
+                                                       if let Some((mut new_outpoints, new_outputs)) = self.check_spend_holder_transaction(&tx, height, &logger) {
+                                                               if !new_outputs.1.is_empty() {
+                                                                       watch_outputs.push(new_outputs);
+                                                               }
+                                                               claimable_outpoints.append(&mut new_outpoints);
+                                                               balance_spendable_csv = Some(self.on_holder_tx_csv);
                                                        }
-                                                       claimable_outpoints.append(&mut new_outpoints);
                                                }
-                                               claimable_outpoints.append(&mut new_outpoints);
                                        }
+                                       let txid = tx.txid();
+                                       self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
+                                               txid,
+                                               height: height,
+                                               event: OnchainEvent::FundingSpendConfirmation {
+                                                       on_local_output_csv: balance_spendable_csv,
+                                               },
+                                       });
                                } else {
                                        if let Some(&commitment_number) = self.counterparty_commitment_txn_on_chain.get(&prevout.txid) {
                                                let (mut new_outpoints, new_outputs_option) = self.check_spend_counterparty_htlc(&tx, commitment_number, height, &logger);
@@ -2049,7 +2357,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        let funding_outp = HolderFundingOutput::build(self.funding_redeemscript.clone());
                        let commitment_package = PackageTemplate::build_package(self.funding_info.0.txid.clone(), self.funding_info.0.index as u32, PackageSolvingData::HolderFundingOutput(funding_outp), self.best_block.height(), false, self.best_block.height());
                        claimable_outpoints.push(commitment_package);
-                       self.pending_monitor_events.push(MonitorEvent::CommitmentTxBroadcasted(self.funding_info.0));
+                       self.pending_monitor_events.push(MonitorEvent::CommitmentTxConfirmed(self.funding_info.0));
                        let commitment_tx = self.onchain_tx_handler.get_fully_signed_holder_tx(&self.funding_redeemscript);
                        self.holder_tx_signed = true;
                        // Because we're broadcasting a commitment transaction, we should construct the package
@@ -2081,7 +2389,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        .iter()
                        .filter_map(|entry| match &entry.event {
                                OnchainEvent::HTLCUpdate { source, .. } => Some(source),
-                               OnchainEvent::MaturingOutput { .. } => None,
+                               _ => None,
                        })
                        .collect();
                #[cfg(debug_assertions)]
@@ -2090,7 +2398,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                // Produce actionable events from on-chain events having reached their threshold.
                for entry in onchain_events_reaching_threshold_conf.drain(..) {
                        match entry.event {
-                               OnchainEvent::HTLCUpdate { ref source, payment_hash, onchain_value_satoshis } => {
+                               OnchainEvent::HTLCUpdate { ref source, payment_hash, onchain_value_satoshis, input_idx } => {
                                        // Check for duplicate HTLC resolutions.
                                        #[cfg(debug_assertions)]
                                        {
@@ -2114,13 +2422,22 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                source: source.clone(),
                                                onchain_value_satoshis,
                                        }));
+                                       if let Some(idx) = input_idx {
+                                               self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { input_idx: idx, payment_preimage: None });
+                                       }
                                },
                                OnchainEvent::MaturingOutput { descriptor } => {
                                        log_debug!(logger, "Descriptor {} has got enough confirmations to be passed upstream", log_spendable!(descriptor));
                                        self.pending_events.push(Event::SpendableOutputs {
                                                outputs: vec![descriptor]
                                        });
-                               }
+                               },
+                               OnchainEvent::HTLCSpendConfirmation { input_idx, preimage, .. } => {
+                                       self.htlcs_resolved_on_chain.push(IrrevocablyResolvedHTLC { input_idx, payment_preimage: preimage });
+                               },
+                               OnchainEvent::FundingSpendConfirmation { .. } => {
+                                       self.funding_spend_confirmed = Some(entry.txid);
+                               },
                        }
                }
 
@@ -2298,15 +2615,32 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        let revocation_sig_claim = (input.witness.len() == 3 && HTLCType::scriptlen_to_htlctype(input.witness[2].len()) == Some(HTLCType::OfferedHTLC) && input.witness[1].len() == 33)
                                || (input.witness.len() == 3 && HTLCType::scriptlen_to_htlctype(input.witness[2].len()) == Some(HTLCType::AcceptedHTLC) && input.witness[1].len() == 33);
                        let accepted_preimage_claim = input.witness.len() == 5 && HTLCType::scriptlen_to_htlctype(input.witness[4].len()) == Some(HTLCType::AcceptedHTLC);
-                       let offered_preimage_claim = input.witness.len() == 3 && HTLCType::scriptlen_to_htlctype(input.witness[2].len()) == Some(HTLCType::OfferedHTLC);
+                       let accepted_timeout_claim = input.witness.len() == 3 && HTLCType::scriptlen_to_htlctype(input.witness[2].len()) == Some(HTLCType::AcceptedHTLC) && !revocation_sig_claim;
+                       let offered_preimage_claim = input.witness.len() == 3 && HTLCType::scriptlen_to_htlctype(input.witness[2].len()) == Some(HTLCType::OfferedHTLC) && !revocation_sig_claim;
+                       let offered_timeout_claim = input.witness.len() == 5 && HTLCType::scriptlen_to_htlctype(input.witness[4].len()) == Some(HTLCType::OfferedHTLC);
+
+                       let mut payment_preimage = PaymentPreimage([0; 32]);
+                       if accepted_preimage_claim {
+                               payment_preimage.0.copy_from_slice(&input.witness[3]);
+                       } else if offered_preimage_claim {
+                               payment_preimage.0.copy_from_slice(&input.witness[1]);
+                       }
 
                        macro_rules! log_claim {
                                ($tx_info: expr, $holder_tx: expr, $htlc: expr, $source_avail: expr) => {
-                                       // We found the output in question, but aren't failing it backwards
-                                       // as we have no corresponding source and no valid counterparty commitment txid
-                                       // to try a weak source binding with same-hash, same-value still-valid offered HTLC.
-                                       // This implies either it is an inbound HTLC or an outbound HTLC on a revoked transaction.
                                        let outbound_htlc = $holder_tx == $htlc.offered;
+                                       // HTLCs must either be claimed by a matching script type or through the
+                                       // revocation path:
+                                       #[cfg(not(fuzzing))] // Note that the fuzzer is not bound by pesky things like "signatures"
+                                       debug_assert!(!$htlc.offered || offered_preimage_claim || offered_timeout_claim || revocation_sig_claim);
+                                       #[cfg(not(fuzzing))] // Note that the fuzzer is not bound by pesky things like "signatures"
+                                       debug_assert!($htlc.offered || accepted_preimage_claim || accepted_timeout_claim || revocation_sig_claim);
+                                       // Further, only exactly one of the possible spend paths should have been
+                                       // matched by any HTLC spend:
+                                       #[cfg(not(fuzzing))] // Note that the fuzzer is not bound by pesky things like "signatures"
+                                       debug_assert_eq!(accepted_preimage_claim as u8 + accepted_timeout_claim as u8 +
+                                                        offered_preimage_claim as u8 + offered_timeout_claim as u8 +
+                                                        revocation_sig_claim as u8, 1);
                                        if ($holder_tx && revocation_sig_claim) ||
                                                        (outbound_htlc && !$source_avail && (accepted_preimage_claim || offered_preimage_claim)) {
                                                log_error!(logger, "Input spending {} ({}:{}) in {} resolves {} HTLC with payment hash {} with {}!",
@@ -2351,13 +2685,37 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                                // resolve the source HTLC with the original sender.
                                                                payment_data = Some(((*source).clone(), htlc_output.payment_hash, htlc_output.amount_msat));
                                                        } else if !$holder_tx {
-                                                                       check_htlc_valid_counterparty!(self.current_counterparty_commitment_txid, htlc_output);
+                                                               check_htlc_valid_counterparty!(self.current_counterparty_commitment_txid, htlc_output);
                                                                if payment_data.is_none() {
                                                                        check_htlc_valid_counterparty!(self.prev_counterparty_commitment_txid, htlc_output);
                                                                }
                                                        }
                                                        if payment_data.is_none() {
                                                                log_claim!($tx_info, $holder_tx, htlc_output, false);
+                                                               let outbound_htlc = $holder_tx == htlc_output.offered;
+                                                               if !outbound_htlc || revocation_sig_claim {
+                                                                       self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
+                                                                               txid: tx.txid(), height,
+                                                                               event: OnchainEvent::HTLCSpendConfirmation {
+                                                                                       input_idx: input.previous_output.vout,
+                                                                                       preimage: if accepted_preimage_claim || offered_preimage_claim {
+                                                                                               Some(payment_preimage) } else { None },
+                                                                                       // If this is a payment to us (!outbound_htlc, above),
+                                                                                       // wait for the CSV delay before dropping the HTLC from
+                                                                                       // claimable balance if the claim was an HTLC-Success
+                                                                                       // transaction.
+                                                                                       on_to_local_output_csv: if accepted_preimage_claim {
+                                                                                               Some(self.on_holder_tx_csv) } else { None },
+                                                                               },
+                                                                       });
+                                                               } else {
+                                                                       // Outbound claims should always have payment_data, unless
+                                                                       // we've already failed the HTLC as the commitment transaction
+                                                                       // which was broadcasted was revoked. In that case, we should
+                                                                       // spend the HTLC output here immediately, and expose that fact
+                                                                       // as a Balance, something which we do not yet do.
+                                                                       // TODO: Track the above as claimable!
+                                                               }
                                                                continue 'outer_loop;
                                                        }
                                                }
@@ -2383,11 +2741,18 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                        // Check that scan_commitment, above, decided there is some source worth relaying an
                        // HTLC resolution backwards to and figure out whether we learned a preimage from it.
                        if let Some((source, payment_hash, amount_msat)) = payment_data {
-                               let mut payment_preimage = PaymentPreimage([0; 32]);
                                if accepted_preimage_claim {
                                        if !self.pending_monitor_events.iter().any(
                                                |update| if let &MonitorEvent::HTLCEvent(ref upd) = update { upd.source == source } else { false }) {
-                                               payment_preimage.0.copy_from_slice(&input.witness[3]);
+                                               self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
+                                                       txid: tx.txid(),
+                                                       height,
+                                                       event: OnchainEvent::HTLCSpendConfirmation {
+                                                               input_idx: input.previous_output.vout,
+                                                               preimage: Some(payment_preimage),
+                                                               on_to_local_output_csv: None,
+                                                       },
+                                               });
                                                self.pending_monitor_events.push(MonitorEvent::HTLCEvent(HTLCUpdate {
                                                        source,
                                                        payment_preimage: Some(payment_preimage),
@@ -2400,7 +2765,15 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                |update| if let &MonitorEvent::HTLCEvent(ref upd) = update {
                                                        upd.source == source
                                                } else { false }) {
-                                               payment_preimage.0.copy_from_slice(&input.witness[1]);
+                                               self.onchain_events_awaiting_threshold_conf.push(OnchainEventEntry {
+                                                       txid: tx.txid(),
+                                                       height,
+                                                       event: OnchainEvent::HTLCSpendConfirmation {
+                                                               input_idx: input.previous_output.vout,
+                                                               preimage: Some(payment_preimage),
+                                                               on_to_local_output_csv: None,
+                                                       },
+                                               });
                                                self.pending_monitor_events.push(MonitorEvent::HTLCEvent(HTLCUpdate {
                                                        source,
                                                        payment_preimage: Some(payment_preimage),
@@ -2424,6 +2797,7 @@ impl<Signer: Sign> ChannelMonitorImpl<Signer> {
                                                event: OnchainEvent::HTLCUpdate {
                                                        source, payment_hash,
                                                        onchain_value_satoshis: Some(amount_msat / 1000),
+                                                       input_idx: Some(input.previous_output.vout),
                                                },
                                        };
                                        log_info!(logger, "Failing HTLC with payment_hash {} timeout by a spend tx, waiting for confirmation (at height {})", log_bytes!(payment_hash.0), entry.confirmation_threshold());
@@ -2635,7 +3009,7 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                let current_counterparty_commitment_txid = Readable::read(reader)?;
                let prev_counterparty_commitment_txid = Readable::read(reader)?;
 
-               let counterparty_tx_cache = Readable::read(reader)?;
+               let counterparty_commitment_params = Readable::read(reader)?;
                let funding_redeemscript = Readable::read(reader)?;
                let channel_value_satoshis = Readable::read(reader)?;
 
@@ -2708,14 +3082,15 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                        }
                }
 
-               let prev_holder_signed_commitment_tx = match <u8 as Readable>::read(reader)? {
-                       0 => None,
-                       1 => {
-                               Some(Readable::read(reader)?)
-                       },
-                       _ => return Err(DecodeError::InvalidValue),
-               };
-               let current_holder_commitment_tx = Readable::read(reader)?;
+               let mut prev_holder_signed_commitment_tx: Option<HolderSignedTx> =
+                       match <u8 as Readable>::read(reader)? {
+                               0 => None,
+                               1 => {
+                                       Some(Readable::read(reader)?)
+                               },
+                               _ => return Err(DecodeError::InvalidValue),
+                       };
+               let mut current_holder_commitment_tx: HolderSignedTx = Readable::read(reader)?;
 
                let current_counterparty_commitment_number = <U48 as Readable>::read(reader)?.0;
                let current_holder_commitment_number = <U48 as Readable>::read(reader)?.0;
@@ -2735,7 +3110,7 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                for _ in 0..pending_monitor_events_len {
                        let ev = match <u8 as Readable>::read(reader)? {
                                0 => MonitorEvent::HTLCEvent(Readable::read(reader)?),
-                               1 => MonitorEvent::CommitmentTxBroadcasted(funding_info.0),
+                               1 => MonitorEvent::CommitmentTxConfirmed(funding_info.0),
                                _ => return Err(DecodeError::InvalidValue)
                        };
                        pending_monitor_events.push(ev);
@@ -2772,12 +3147,34 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                                return Err(DecodeError::InvalidValue);
                        }
                }
-               let onchain_tx_handler = ReadableArgs::read(reader, keys_manager)?;
+               let onchain_tx_handler: OnchainTxHandler<Signer> = ReadableArgs::read(reader, keys_manager)?;
 
                let lockdown_from_offchain = Readable::read(reader)?;
                let holder_tx_signed = Readable::read(reader)?;
 
-               read_tlv_fields!(reader, {});
+               if let Some(prev_commitment_tx) = prev_holder_signed_commitment_tx.as_mut() {
+                       let prev_holder_value = onchain_tx_handler.get_prev_holder_commitment_to_self_value();
+                       if prev_holder_value.is_none() { return Err(DecodeError::InvalidValue); }
+                       if prev_commitment_tx.to_self_value_sat == u64::max_value() {
+                               prev_commitment_tx.to_self_value_sat = prev_holder_value.unwrap();
+                       } else if prev_commitment_tx.to_self_value_sat != prev_holder_value.unwrap() {
+                               return Err(DecodeError::InvalidValue);
+                       }
+               }
+
+               let cur_holder_value = onchain_tx_handler.get_cur_holder_commitment_to_self_value();
+               if current_holder_commitment_tx.to_self_value_sat == u64::max_value() {
+                       current_holder_commitment_tx.to_self_value_sat = cur_holder_value;
+               } else if current_holder_commitment_tx.to_self_value_sat != cur_holder_value {
+                       return Err(DecodeError::InvalidValue);
+               }
+
+               let mut funding_spend_confirmed = None;
+               let mut htlcs_resolved_on_chain = Some(Vec::new());
+               read_tlv_fields!(reader, {
+                       (1, funding_spend_confirmed, option),
+                       (3, htlcs_resolved_on_chain, vec_type),
+               });
 
                let mut secp_ctx = Secp256k1::new();
                secp_ctx.seeded_randomize(&keys_manager.get_secure_random_bytes());
@@ -2798,7 +3195,7 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
                                current_counterparty_commitment_txid,
                                prev_counterparty_commitment_txid,
 
-                               counterparty_tx_cache,
+                               counterparty_commitment_params,
                                funding_redeemscript,
                                channel_value_satoshis,
                                their_cur_revocation_points,
@@ -2826,6 +3223,8 @@ impl<'a, Signer: Sign, K: KeysInterface<Signer = Signer>> ReadableArgs<&'a K>
 
                                lockdown_from_offchain,
                                holder_tx_signed,
+                               funding_spend_confirmed,
+                               htlcs_resolved_on_chain: htlcs_resolved_on_chain.unwrap(),
 
                                best_block,
 
index 44a03d09f8e717019c19c070159a7b7556b24723..2da9e575047f68ec805fe0089642cdd90169ce92 100644 (file)
@@ -34,7 +34,7 @@ use util::ser::{Writeable, Writer, Readable};
 
 use chain::transaction::OutPoint;
 use ln::chan_utils;
-use ln::chan_utils::{HTLCOutputInCommitment, make_funding_redeemscript, ChannelPublicKeys, HolderCommitmentTransaction, ChannelTransactionParameters, CommitmentTransaction};
+use ln::chan_utils::{HTLCOutputInCommitment, make_funding_redeemscript, ChannelPublicKeys, HolderCommitmentTransaction, ChannelTransactionParameters, CommitmentTransaction, ClosingTransaction};
 use ln::msgs::UnsignedChannelAnnouncement;
 use ln::script::ShutdownScript;
 
@@ -322,7 +322,7 @@ pub trait BaseSign {
        ///
        /// Note that, due to rounding, there may be one "missing" satoshi, and either party may have
        /// chosen to forgo their output as dust.
-       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
+       fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()>;
 
        /// Signs a channel announcement message with our funding key, proving it comes from one
        /// of the channel participants.
@@ -671,17 +671,10 @@ impl BaseSign for InMemorySigner {
                Err(())
        }
 
-       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
-               if closing_tx.input.len() != 1 { return Err(()); }
-               if closing_tx.input[0].witness.len() != 0 { return Err(()); }
-               if closing_tx.output.len() > 2 { return Err(()); }
-
+       fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
                let funding_pubkey = PublicKey::from_secret_key(secp_ctx, &self.funding_key);
                let channel_funding_redeemscript = make_funding_redeemscript(&funding_pubkey, &self.counterparty_pubkeys().funding_pubkey);
-
-               let sighash = hash_to_message!(&bip143::SigHashCache::new(closing_tx)
-                       .signature_hash(0, &channel_funding_redeemscript, self.channel_value_satoshis, SigHashType::All)[..]);
-               Ok(secp_ctx.sign(&sighash, &self.funding_key))
+               Ok(closing_tx.trust().sign(&self.funding_key, &channel_funding_redeemscript, self.channel_value_satoshis, secp_ctx))
        }
 
        fn sign_channel_announcement(&self, msg: &UnsignedChannelAnnouncement, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
index d6777cc5c8cfd494409f5832f29efb3699bc21ab..b4f5438adfb144645aa3acff008580a7fd74a8fc 100644 (file)
@@ -365,6 +365,14 @@ impl<ChannelSigner: Sign> OnchainTxHandler<ChannelSigner> {
                }
        }
 
+       pub(crate) fn get_prev_holder_commitment_to_self_value(&self) -> Option<u64> {
+               self.prev_holder_commitment.as_ref().map(|commitment| commitment.to_broadcaster_value_sat())
+       }
+
+       pub(crate) fn get_cur_holder_commitment_to_self_value(&self) -> u64 {
+               self.holder_commitment.to_broadcaster_value_sat()
+       }
+
        /// Lightning security model (i.e being able to redeem/timeout HTLC or penalize coutnerparty onchain) lays on the assumption of claim transactions getting confirmed before timelock expiration
        /// (CSV or CLTV following cases). In case of high-fee spikes, claim tx may stuck in the mempool, so you need to bump its feerate quickly using Replace-By-Fee or Child-Pay-For-Parent.
        /// Panics if there are signing errors, because signing operations in reaction to on-chain events
index 502eb895b2683e4ad3b7fa5bc34f367111781ede..0219ebbe8cad9631e1f2fef8c7b05600828e9f24 100644 (file)
@@ -75,7 +75,7 @@ impl OutPoint {
        }
 }
 
-impl_writeable!(OutPoint, 0, { txid, index });
+impl_writeable!(OutPoint, { txid, index });
 
 #[cfg(test)]
 mod tests {
index 6bdef4307350d17c158ef2fe88dac121532db532..994e1c276c4427168bca70a6aa1369bc7f1a741f 100644 (file)
@@ -23,7 +23,7 @@ use bitcoin::hash_types::{Txid, PubkeyHash};
 use ln::{PaymentHash, PaymentPreimage};
 use ln::msgs::DecodeError;
 use util::ser::{Readable, Writeable, Writer};
-use util::byte_utils;
+use util::{byte_utils, transaction_utils};
 
 use bitcoin::hash_types::WPubkeyHash;
 use bitcoin::secp256k1::key::{SecretKey, PublicKey};
@@ -80,6 +80,50 @@ pub fn build_commitment_secret(commitment_seed: &[u8; 32], idx: u64) -> [u8; 32]
        res
 }
 
+/// Build a closing transaction
+pub fn build_closing_transaction(to_holder_value_sat: u64, to_counterparty_value_sat: u64, to_holder_script: Script, to_counterparty_script: Script, funding_outpoint: OutPoint) -> Transaction {
+       let txins = {
+               let mut ins: Vec<TxIn> = Vec::new();
+               ins.push(TxIn {
+                       previous_output: funding_outpoint,
+                       script_sig: Script::new(),
+                       sequence: 0xffffffff,
+                       witness: Vec::new(),
+               });
+               ins
+       };
+
+       let mut txouts: Vec<(TxOut, ())> = Vec::new();
+
+       if to_counterparty_value_sat > 0 {
+               txouts.push((TxOut {
+                       script_pubkey: to_counterparty_script,
+                       value: to_counterparty_value_sat
+               }, ()));
+       }
+
+       if to_holder_value_sat > 0 {
+               txouts.push((TxOut {
+                       script_pubkey: to_holder_script,
+                       value: to_holder_value_sat
+               }, ()));
+       }
+
+       transaction_utils::sort_outputs(&mut txouts, |_, _| { cmp::Ordering::Equal }); // Ordering doesnt matter if they used our pubkey...
+
+       let mut outputs: Vec<TxOut> = Vec::new();
+       for out in txouts.drain(..) {
+               outputs.push(out.0);
+       }
+
+       Transaction {
+               version: 2,
+               lock_time: 0,
+               input: txins,
+               output: outputs,
+       }
+}
+
 /// Implements the per-commitment secret storage scheme from
 /// [BOLT 3](https://github.com/lightningnetwork/lightning-rfc/blob/dcbf8583976df087c79c3ce0b535311212e6812d/03-transactions.md#efficient-per-commitment-secret-storage).
 ///
@@ -846,7 +890,130 @@ impl BuiltCommitmentTransaction {
        }
 }
 
-/// This class tracks the per-transaction information needed to build a commitment transaction and to
+/// This class tracks the per-transaction information needed to build a closing transaction and will
+/// actually build it and sign.
+///
+/// This class can be used inside a signer implementation to generate a signature given the relevant
+/// secret key.
+pub struct ClosingTransaction {
+       to_holder_value_sat: u64,
+       to_counterparty_value_sat: u64,
+       to_holder_script: Script,
+       to_counterparty_script: Script,
+       built: Transaction,
+}
+
+impl ClosingTransaction {
+       /// Construct an object of the class
+       pub fn new(
+               to_holder_value_sat: u64,
+               to_counterparty_value_sat: u64,
+               to_holder_script: Script,
+               to_counterparty_script: Script,
+               funding_outpoint: OutPoint,
+       ) -> Self {
+               let built = build_closing_transaction(
+                       to_holder_value_sat, to_counterparty_value_sat,
+                       to_holder_script.clone(), to_counterparty_script.clone(),
+                       funding_outpoint
+               );
+               ClosingTransaction {
+                       to_holder_value_sat,
+                       to_counterparty_value_sat,
+                       to_holder_script,
+                       to_counterparty_script,
+                       built
+               }
+       }
+
+       /// Trust our pre-built transaction.
+       ///
+       /// Applies a wrapper which allows access to the transaction.
+       ///
+       /// This should only be used if you fully trust the builder of this object. It should not
+       /// be used by an external signer - instead use the verify function.
+       pub fn trust(&self) -> TrustedClosingTransaction {
+               TrustedClosingTransaction { inner: self }
+       }
+
+       /// Verify our pre-built transaction.
+       ///
+       /// Applies a wrapper which allows access to the transaction.
+       ///
+       /// An external validating signer must call this method before signing
+       /// or using the built transaction.
+       pub fn verify(&self, funding_outpoint: OutPoint) -> Result<TrustedClosingTransaction, ()> {
+               let built = build_closing_transaction(
+                       self.to_holder_value_sat, self.to_counterparty_value_sat,
+                       self.to_holder_script.clone(), self.to_counterparty_script.clone(),
+                       funding_outpoint
+               );
+               if self.built != built {
+                       return Err(())
+               }
+               Ok(TrustedClosingTransaction { inner: self })
+       }
+
+       /// The value to be sent to the holder, or zero if the output will be omitted
+       pub fn to_holder_value_sat(&self) -> u64 {
+               self.to_holder_value_sat
+       }
+
+       /// The value to be sent to the counterparty, or zero if the output will be omitted
+       pub fn to_counterparty_value_sat(&self) -> u64 {
+               self.to_counterparty_value_sat
+       }
+
+       /// The destination of the holder's output
+       pub fn to_holder_script(&self) -> &Script {
+               &self.to_holder_script
+       }
+
+       /// The destination of the counterparty's output
+       pub fn to_counterparty_script(&self) -> &Script {
+               &self.to_counterparty_script
+       }
+}
+
+/// A wrapper on ClosingTransaction indicating that the built bitcoin
+/// transaction is trusted.
+///
+/// See trust() and verify() functions on CommitmentTransaction.
+///
+/// This structure implements Deref.
+pub struct TrustedClosingTransaction<'a> {
+       inner: &'a ClosingTransaction,
+}
+
+impl<'a> Deref for TrustedClosingTransaction<'a> {
+       type Target = ClosingTransaction;
+
+       fn deref(&self) -> &Self::Target { self.inner }
+}
+
+impl<'a> TrustedClosingTransaction<'a> {
+       /// The pre-built Bitcoin commitment transaction
+       pub fn built_transaction(&self) -> &Transaction {
+               &self.inner.built
+       }
+
+       /// Get the SIGHASH_ALL sighash value of the transaction.
+       ///
+       /// This can be used to verify a signature.
+       pub fn get_sighash_all(&self, funding_redeemscript: &Script, channel_value_satoshis: u64) -> Message {
+               let sighash = &bip143::SigHashCache::new(&self.inner.built).signature_hash(0, funding_redeemscript, channel_value_satoshis, SigHashType::All)[..];
+               hash_to_message!(sighash)
+       }
+
+       /// Sign a transaction, either because we are counter-signing the counterparty's transaction or
+       /// because we are about to broadcast a holder transaction.
+       pub fn sign<T: secp256k1::Signing>(&self, funding_key: &SecretKey, funding_redeemscript: &Script, channel_value_satoshis: u64, secp_ctx: &Secp256k1<T>) -> Signature {
+               let sighash = self.get_sighash_all(funding_redeemscript, channel_value_satoshis);
+               secp_ctx.sign(&sighash, funding_key)
+       }
+}
+
+/// This class tracks the per-transaction information needed to build a commitment transaction and will
 /// actually build it and sign.  It is used for holder transactions that we sign only when needed
 /// and for transactions we sign for the counterparty.
 ///
@@ -1110,7 +1277,7 @@ impl CommitmentTransaction {
        /// Applies a wrapper which allows access to these fields.
        ///
        /// This should only be used if you fully trust the builder of this object.  It should not
-       ///     be used by an external signer - instead use the verify function.
+       /// be used by an external signer - instead use the verify function.
        pub fn trust(&self) -> TrustedCommitmentTransaction {
                TrustedCommitmentTransaction { inner: self }
        }
index 70bffeeb201bee2215377db323b66de821660d8f..c3262f17374327b42bc40c987350a05a6d8441d9 100644 (file)
@@ -28,7 +28,7 @@ use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
 use routing::router::get_route;
 use util::config::UserConfig;
 use util::enforcing_trait_impls::EnforcingSigner;
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason};
 use util::errors::APIError;
 use util::ser::{ReadableArgs, Writeable};
 use util::test_utils::TestBroadcaster;
@@ -62,7 +62,7 @@ fn do_test_simple_monitor_permanent_update_fail(persister_fail: bool) {
                false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::PermanentFailure))
        }
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), true, APIError::ChannelUnavailable {..}, {});
        check_added_monitors!(nodes[0], 2);
 
@@ -78,9 +78,10 @@ fn do_test_simple_monitor_permanent_update_fail(persister_fail: bool) {
        };
 
        // TODO: Once we hit the chain with the failure transaction we should check that we get a
-       // PaymentFailed event
+       // PaymentPathFailed event
 
        assert_eq!(nodes[0].node.list_channels().len(), 0);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "ChannelMonitor storage failure".to_string() });
 }
 
 #[test]
@@ -186,7 +187,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail
 
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), false, APIError::MonitorUpdateFailed, {});
                check_added_monitors!(nodes[0], 1);
        }
@@ -245,7 +246,7 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail
                        false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure))
                }
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateFailed, {});
                check_added_monitors!(nodes[0], 1);
        }
@@ -266,9 +267,10 @@ fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail
        check_closed_broadcast!(nodes[0], true);
 
        // TODO: Once we hit the chain with the failure transaction we should check that we get a
-       // PaymentFailed event
+       // PaymentPathFailed event
 
        assert_eq!(nodes[0].node.list_channels().len(), 0);
+       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 }
 
 #[test]
@@ -315,7 +317,7 @@ fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
        {
                *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateFailed, {});
                check_added_monitors!(nodes[0], 1);
        }
@@ -652,7 +654,7 @@ fn test_monitor_update_fail_cs() {
        let (payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -747,7 +749,7 @@ fn test_monitor_update_fail_no_rebroadcast() {
        let (payment_preimage_1, our_payment_hash, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash, &Some(payment_secret_1)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -798,7 +800,7 @@ fn test_monitor_update_raa_while_paused() {
        let (payment_preimage_1, our_payment_hash_1, our_payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash_1, &Some(our_payment_secret_1)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -807,7 +809,7 @@ fn test_monitor_update_raa_while_paused() {
        let (payment_preimage_2, our_payment_hash_2, our_payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
        {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, our_payment_hash_2, &Some(our_payment_secret_2)).unwrap();
                check_added_monitors!(nodes[1], 1);
        }
@@ -899,7 +901,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[2]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -926,7 +928,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
        let (_, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[2]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -947,7 +949,7 @@ fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
                // Try to route another payment backwards from 2 to make sure 1 holds off on responding
                let (payment_preimage_4, payment_hash_4, payment_secret_4) = get_payment_preimage_hash!(nodes[0]);
                let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
-               let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[2].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4)).unwrap();
                check_added_monitors!(nodes[2], 1);
 
@@ -1248,7 +1250,7 @@ fn raa_no_response_awaiting_raa_state() {
        // generation during RAA while in monitor-update-failed state.
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
                check_added_monitors!(nodes[0], 1);
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
@@ -1302,7 +1304,7 @@ fn raa_no_response_awaiting_raa_state() {
        // commitment transaction states) whereas here we can explicitly check for it.
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
                check_added_monitors!(nodes[0], 0);
                assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
@@ -1394,7 +1396,7 @@ fn claim_while_disconnected_monitor_update_fail() {
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1490,7 +1492,7 @@ fn monitor_failed_no_reestablish_response() {
        let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1566,7 +1568,7 @@ fn first_message_on_recv_ordering() {
        let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1591,7 +1593,7 @@ fn first_message_on_recv_ordering() {
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1678,7 +1680,7 @@ fn test_monitor_update_fail_claim() {
        let route;
        {
                let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
-               route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1_000_000, TEST_FINAL_CLTV, &logger).unwrap();
+               route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1_000_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
@@ -1788,7 +1790,7 @@ fn test_monitor_update_on_pending_forwards() {
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
        {
                let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
-               let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[2], 1);
        }
@@ -1817,7 +1819,7 @@ fn test_monitor_update_on_pending_forwards() {
 
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 2);
-       if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
+       if let Event::PaymentPathFailed { payment_hash, rejected_by_dest, .. } = events[0] {
                assert_eq!(payment_hash, payment_hash_1);
                assert!(rejected_by_dest);
        } else { panic!("Unexpected event!"); }
@@ -1851,7 +1853,7 @@ fn monitor_update_claim_fail_no_response() {
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1985,6 +1987,8 @@ fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf:
 
        send_payment(&nodes[0], &[&nodes[1]], 8000000);
        close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -2010,7 +2014,7 @@ fn test_path_paused_mpp() {
        let logger = test_utils::TestLogger::new();
 
        let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[3]);
-       let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
 
        // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
        let path = route.paths[0].clone();
@@ -2075,7 +2079,7 @@ fn test_pending_update_fee_ack_on_reconnect() {
        send_payment(&nodes[0], &[&nodes[1]], 100_000_00);
 
        let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[0]);
-       let route = get_route(&nodes[1].node.get_our_node_id(), &nodes[1].net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&nodes[1].node.get_our_node_id(), &nodes[1].net_graph_msg_handler.network_graph,
                &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1_000_000, TEST_FINAL_CLTV, nodes[1].logger).unwrap();
        nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
        check_added_monitors!(nodes[1], 1);
@@ -2283,7 +2287,7 @@ fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
 
        let route = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, None, &Vec::new(), 100000, TEST_FINAL_CLTV, nodes[0].logger).unwrap()
+               get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), None, None, &Vec::new(), 100000, TEST_FINAL_CLTV, nodes[0].logger).unwrap()
        };
 
        nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
@@ -2470,7 +2474,7 @@ fn do_test_reconnect_dup_htlc_claims(htlc_status: HTLCStatusAtDupClaim, second_f
                // In order to get the HTLC claim into the holding cell at nodes[1], we need nodes[1] to be
                // awaiting a remote revoke_and_ack from nodes[0].
                let (_, second_payment_hash, second_payment_secret) = get_payment_preimage_hash!(nodes[1]);
-               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(),
+               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph,
                        &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, nodes[1].logger).unwrap();
                nodes[0].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
@@ -2610,6 +2614,8 @@ fn test_temporary_error_during_shutdown() {
        assert_eq!(txn_a, txn_b);
        assert_eq!(txn_a.len(), 1);
        check_spends!(txn_a[0], funding_tx);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -2630,6 +2636,7 @@ fn test_permanent_error_during_sending_shutdown() {
        assert!(nodes[0].node.close_channel(&channel_id).is_ok());
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 2);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "ChannelMonitor storage failure".to_string() });
 }
 
 #[test]
@@ -2652,4 +2659,102 @@ fn test_permanent_error_during_handling_shutdown() {
        nodes[1].node.handle_shutdown(&nodes[0].node.get_our_node_id(), &InitFeatures::known(), &shutdown);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 2);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "ChannelMonitor storage failure".to_string() });
+}
+
+#[test]
+fn double_temp_error() {
+       // Test that it's OK to have multiple `ChainMonitor::update_channel` calls fail in a row.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, channel_id, _) = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+
+       let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
+       let (payment_preimage_2, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
+
+       *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+       // `claim_funds` results in a ChannelMonitorUpdate.
+       assert!(nodes[1].node.claim_funds(payment_preimage_1));
+       check_added_monitors!(nodes[1], 1);
+       let (funding_tx, latest_update_1) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+
+       *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
+       // Previously, this would've panicked due to a double-call to `Channel::monitor_update_failed`,
+       // which had some asserts that prevented it from being called twice.
+       assert!(nodes[1].node.claim_funds(payment_preimage_2));
+       check_added_monitors!(nodes[1], 1);
+       *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
+
+       let (_, latest_update_2) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
+       nodes[1].node.channel_monitor_updated(&funding_tx, latest_update_1);
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+       check_added_monitors!(nodes[1], 0);
+       nodes[1].node.channel_monitor_updated(&funding_tx, latest_update_2);
+
+       // Complete the first HTLC.
+       let events = nodes[1].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       let (update_fulfill_1, commitment_signed_b1, node_id) = {
+               match &events[0] {
+                       &MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+                               assert!(update_add_htlcs.is_empty());
+                               assert_eq!(update_fulfill_htlcs.len(), 1);
+                               assert!(update_fail_htlcs.is_empty());
+                               assert!(update_fail_malformed_htlcs.is_empty());
+                               assert!(update_fee.is_none());
+                               (update_fulfill_htlcs[0].clone(), commitment_signed.clone(), node_id.clone())
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+       };
+       assert_eq!(node_id, nodes[0].node.get_our_node_id());
+       nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_1);
+       check_added_monitors!(nodes[0], 0);
+       expect_payment_sent!(nodes[0], payment_preimage_1);
+       nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &commitment_signed_b1);
+       check_added_monitors!(nodes[0], 1);
+       nodes[0].node.process_pending_htlc_forwards();
+       let (raa_a1, commitment_signed_a1) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+       check_added_monitors!(nodes[1], 0);
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+       nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &raa_a1);
+       check_added_monitors!(nodes[1], 1);
+       nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &commitment_signed_a1);
+       check_added_monitors!(nodes[1], 1);
+
+       // Complete the second HTLC.
+       let ((update_fulfill_2, commitment_signed_b2), raa_b2) = {
+               let events = nodes[1].node.get_and_clear_pending_msg_events();
+               assert_eq!(events.len(), 2);
+               (match &events[0] {
+                       MessageSendEvent::UpdateHTLCs { node_id, updates } => {
+                               assert_eq!(*node_id, nodes[0].node.get_our_node_id());
+                               assert!(updates.update_add_htlcs.is_empty());
+                               assert!(updates.update_fail_htlcs.is_empty());
+                               assert!(updates.update_fail_malformed_htlcs.is_empty());
+                               assert!(updates.update_fee.is_none());
+                               assert_eq!(updates.update_fulfill_htlcs.len(), 1);
+                               (updates.update_fulfill_htlcs[0].clone(), updates.commitment_signed.clone())
+                       },
+                       _ => panic!("Unexpected event"),
+               },
+                match events[1] {
+                        MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
+                                assert_eq!(*node_id, nodes[0].node.get_our_node_id());
+                                (*msg).clone()
+                        },
+                        _ => panic!("Unexpected event"),
+                })
+       };
+       nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa_b2);
+       check_added_monitors!(nodes[0], 1);
+
+       nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_2);
+       check_added_monitors!(nodes[0], 0);
+       assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
+       commitment_signed_dance!(nodes[0], nodes[1], commitment_signed_b2, false);
+       expect_payment_sent!(nodes[0], payment_preimage_2);
 }
index 58384f3855e6ceddd7a5a4da646a5e1454e0c45c..11995c50d7fa0e1314fe3edcfadaca4993b7ec10 100644 (file)
@@ -8,7 +8,7 @@
 // licenses.
 
 use bitcoin::blockdata::script::{Script,Builder};
-use bitcoin::blockdata::transaction::{TxIn, TxOut, Transaction, SigHashType};
+use bitcoin::blockdata::transaction::{Transaction, SigHashType};
 use bitcoin::util::bip143;
 use bitcoin::consensus::encode;
 
@@ -26,16 +26,15 @@ use ln::{PaymentPreimage, PaymentHash};
 use ln::features::{ChannelFeatures, InitFeatures};
 use ln::msgs;
 use ln::msgs::{DecodeError, OptionalField, DataLossProtect};
-use ln::script::ShutdownScript;
-use ln::channelmanager::{PendingHTLCStatus, HTLCSource, HTLCFailReason, HTLCFailureMsg, PendingHTLCInfo, RAACommitmentOrder, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA, MAX_LOCAL_BREAKDOWN_TIMEOUT};
-use ln::chan_utils::{CounterpartyCommitmentSecrets, TxCreationKeys, HTLCOutputInCommitment, HTLC_SUCCESS_TX_WEIGHT, HTLC_TIMEOUT_TX_WEIGHT, make_funding_redeemscript, ChannelPublicKeys, CommitmentTransaction, HolderCommitmentTransaction, ChannelTransactionParameters, CounterpartyChannelTransactionParameters, MAX_HTLCS, get_commitment_transaction_number_obscure_factor};
+use ln::script::{self, ShutdownScript};
+use ln::channelmanager::{CounterpartyForwardingInfo, PendingHTLCStatus, HTLCSource, HTLCFailReason, HTLCFailureMsg, PendingHTLCInfo, RAACommitmentOrder, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA, MAX_LOCAL_BREAKDOWN_TIMEOUT};
+use ln::chan_utils::{CounterpartyCommitmentSecrets, TxCreationKeys, HTLCOutputInCommitment, HTLC_SUCCESS_TX_WEIGHT, HTLC_TIMEOUT_TX_WEIGHT, make_funding_redeemscript, ChannelPublicKeys, CommitmentTransaction, HolderCommitmentTransaction, ChannelTransactionParameters, CounterpartyChannelTransactionParameters, MAX_HTLCS, get_commitment_transaction_number_obscure_factor, ClosingTransaction};
 use ln::chan_utils;
 use chain::BestBlock;
 use chain::chaininterface::{FeeEstimator,ConfirmationTarget};
 use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdate, ChannelMonitorUpdateStep, HTLC_FAIL_BACK_BUFFER};
 use chain::transaction::{OutPoint, TransactionData};
 use chain::keysinterface::{Sign, KeysInterface};
-use util::transaction_utils;
 use util::ser::{Readable, ReadableArgs, Writeable, Writer, VecWriter};
 use util::logger::Logger;
 use util::errors::APIError;
@@ -45,7 +44,6 @@ use util::scid_utils::scid_from_parts;
 use io;
 use prelude::*;
 use core::{cmp,mem,fmt};
-use core::convert::TryFrom;
 use core::ops::Deref;
 #[cfg(any(test, feature = "fuzztarget", debug_assertions))]
 use sync::Mutex;
@@ -311,19 +309,6 @@ impl HTLCCandidate {
        }
 }
 
-/// Information needed for constructing an invoice route hint for this channel.
-#[derive(Clone, Debug, PartialEq)]
-pub struct CounterpartyForwardingInfo {
-       /// Base routing fee in millisatoshis.
-       pub fee_base_msat: u32,
-       /// Amount in millionths of a satoshi the channel will charge per transferred satoshi.
-       pub fee_proportional_millionths: u32,
-       /// The minimum difference in cltv_expiry between an ingoing HTLC and its outgoing counterpart,
-       /// such that the outgoing HTLC is forwardable to this counterparty. See `msgs::ChannelUpdate`'s
-       /// `cltv_expiry_delta` for more details.
-       pub cltv_expiry_delta: u16,
-}
-
 /// A return value enum for get_update_fulfill_htlc. See UpdateFulfillCommitFetch variants for
 /// description
 enum UpdateFulfillFetch {
@@ -455,8 +440,8 @@ pub(super) struct Channel<Signer: Sign> {
        /// closing_signed message and handling it in `maybe_propose_closing_signed`.
        pending_counterparty_closing_signed: Option<msgs::ClosingSigned>,
 
-       /// The minimum and maximum absolute fee we are willing to place on the closing transaction.
-       /// These are set once we reach `closing_negotiation_ready`.
+       /// The minimum and maximum absolute fee, in satoshis, we are willing to place on the closing
+       /// transaction. These are set once we reach `closing_negotiation_ready`.
        #[cfg(test)]
        pub(crate) closing_fee_limits: Option<(u64, u64)>,
        #[cfg(not(test))]
@@ -569,21 +554,24 @@ pub const ANCHOR_OUTPUT_VALUE_SATOSHI: u64 = 330;
 /// it's 2^24.
 pub const MAX_FUNDING_SATOSHIS: u64 = 1 << 24;
 
-/// Maximum counterparty `dust_limit_satoshis` allowed. 2 * standard dust threshold on p2wsh output
-/// Scales up on Bitcoin Core's proceeding policy with dust outputs. A typical p2wsh output is 43
-/// bytes to which Core's `GetDustThreshold()` sums up a minimal spend of 67 bytes (even if
-/// a p2wsh witnessScript might be *effectively* smaller), `dustRelayFee` is set to 3000sat/kb, thus
-/// 110 * 3000 / 1000 = 330. Per-protocol rules, all time-sensitive outputs are p2wsh, a value of
-/// 330 sats is the lower bound desired to ensure good propagation of transactions. We give a bit
-/// of margin to our counterparty and pick up 660 satoshis as an accepted `dust_limit_satoshis`
-/// upper bound to avoid negotiation conflicts with other implementations.
-pub const MAX_DUST_LIMIT_SATOSHIS: u64 = 2 * 330;
-
-/// A typical p2wsh output is 43 bytes to which Core's `GetDustThreshold()` sums up a minimal
-/// spend of 67 bytes (even if a p2wsh witnessScript might be *effectively* smaller), `dustRelayFee`
-/// is set to 3000sat/kb, thus 110 * 3000 / 1000 = 330. Per-protocol rules, all time-sensitive outputs
-/// are p2wsh, a value of 330 sats is the lower bound desired to ensure good propagation of transactions.
-pub const MIN_DUST_LIMIT_SATOSHIS: u64 = 330;
+/// The maximum network dust limit for standard script formats. This currently represents the
+/// minimum output value for a P2SH output before Bitcoin Core 22 considers the entire
+/// transaction non-standard and thus refuses to relay it.
+/// We also use this as the maximum counterparty `dust_limit_satoshis` allowed, given many
+/// implementations use this value for their dust limit today.
+pub const MAX_STD_OUTPUT_DUST_LIMIT_SATOSHIS: u64 = 546;
+
+/// The maximum channel dust limit we will accept from our counterparty.
+pub const MAX_CHAN_DUST_LIMIT_SATOSHIS: u64 = MAX_STD_OUTPUT_DUST_LIMIT_SATOSHIS;
+
+/// The dust limit is used for both the commitment transaction outputs as well as the closing
+/// transactions. For cooperative closing transactions, we require segwit outputs, though accept
+/// *any* segwit scripts, which are allowed to be up to 42 bytes in length.
+/// In order to avoid having to concern ourselves with standardness during the closing process, we
+/// simply require our counterparty to use a dust limit which will leave any segwit output
+/// standard.
+/// See https://github.com/lightningnetwork/lightning-rfc/issues/905 for more details.
+pub const MIN_CHAN_DUST_LIMIT_SATOSHIS: u64 = 354;
 
 /// Used to return a simple Error back to ChannelManager. Will get converted to a
 /// msgs::ErrorAction::SendErrorMessage or msgs::ErrorAction::IgnoreError as appropriate with our
@@ -650,7 +638,7 @@ impl<Signer: Sign> Channel<Signer> {
                        return Err(APIError::APIMisuseError {err: format!("Configured with an unreasonable our_to_self_delay ({}) putting user funds at risks", holder_selected_contest_delay)});
                }
                let holder_selected_channel_reserve_satoshis = Channel::<Signer>::get_holder_selected_channel_reserve_satoshis(channel_value_satoshis);
-               if holder_selected_channel_reserve_satoshis < MIN_DUST_LIMIT_SATOSHIS {
+               if holder_selected_channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS {
                        return Err(APIError::APIMisuseError { err: format!("Holder selected channel  reserve below implemention limit dust_limit_satoshis {}", holder_selected_channel_reserve_satoshis) });
                }
 
@@ -721,7 +709,7 @@ impl<Signer: Sign> Channel<Signer> {
 
                        feerate_per_kw: feerate,
                        counterparty_dust_limit_satoshis: 0,
-                       holder_dust_limit_satoshis: MIN_DUST_LIMIT_SATOSHIS,
+                       holder_dust_limit_satoshis: MIN_CHAN_DUST_LIMIT_SATOSHIS,
                        counterparty_max_htlc_value_in_flight_msat: 0,
                        counterparty_selected_channel_reserve_satoshis: None, // Filled in in accept_channel
                        counterparty_htlc_minimum_msat: 0,
@@ -855,11 +843,11 @@ impl<Signer: Sign> Channel<Signer> {
                if msg.max_accepted_htlcs < config.peer_channel_config_limits.min_max_accepted_htlcs {
                        return Err(ChannelError::Close(format!("max_accepted_htlcs ({}) is less than the user specified limit ({})", msg.max_accepted_htlcs, config.peer_channel_config_limits.min_max_accepted_htlcs)));
                }
-               if msg.dust_limit_satoshis < MIN_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is less than the implementation limit ({})", msg.dust_limit_satoshis, MIN_DUST_LIMIT_SATOSHIS)));
+               if msg.dust_limit_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is less than the implementation limit ({})", msg.dust_limit_satoshis, MIN_CHAN_DUST_LIMIT_SATOSHIS)));
                }
-               if msg.dust_limit_satoshis >  MAX_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is greater than the implementation limit ({})", msg.dust_limit_satoshis, MAX_DUST_LIMIT_SATOSHIS)));
+               if msg.dust_limit_satoshis >  MAX_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is greater than the implementation limit ({})", msg.dust_limit_satoshis, MAX_CHAN_DUST_LIMIT_SATOSHIS)));
                }
 
                // Convert things into internal flags and prep our state:
@@ -876,11 +864,11 @@ impl<Signer: Sign> Channel<Signer> {
                let background_feerate = fee_estimator.get_est_sat_per_1000_weight(ConfirmationTarget::Background);
 
                let holder_selected_channel_reserve_satoshis = Channel::<Signer>::get_holder_selected_channel_reserve_satoshis(msg.funding_satoshis);
-               if holder_selected_channel_reserve_satoshis < MIN_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("Suitable channel reserve not found. remote_channel_reserve was ({}). dust_limit_satoshis is ({}).", holder_selected_channel_reserve_satoshis, MIN_DUST_LIMIT_SATOSHIS)));
+               if holder_selected_channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("Suitable channel reserve not found. remote_channel_reserve was ({}). dust_limit_satoshis is ({}).", holder_selected_channel_reserve_satoshis, MIN_CHAN_DUST_LIMIT_SATOSHIS)));
                }
-               if msg.channel_reserve_satoshis < MIN_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("channel_reserve_satoshis ({}) is smaller than our dust limit ({})", msg.channel_reserve_satoshis, MIN_DUST_LIMIT_SATOSHIS)));
+               if msg.channel_reserve_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("channel_reserve_satoshis ({}) is smaller than our dust limit ({})", msg.channel_reserve_satoshis, MIN_CHAN_DUST_LIMIT_SATOSHIS)));
                }
                if holder_selected_channel_reserve_satoshis < msg.dust_limit_satoshis {
                        return Err(ChannelError::Close(format!("Dust limit ({}) too high for the channel reserve we require the remote to keep ({})", msg.dust_limit_satoshis, holder_selected_channel_reserve_satoshis)));
@@ -907,10 +895,10 @@ impl<Signer: Sign> Channel<Signer> {
                                        if script.len() == 0 {
                                                None
                                        } else {
-                                               match ShutdownScript::try_from((script.clone(), their_features)) {
-                                                       Ok(shutdown_script) => Some(shutdown_script.into_inner()),
-                                                       Err(_) => return Err(ChannelError::Close(format!("Peer is signaling upfront_shutdown but has provided an unacceptable scriptpubkey format: {}", script))),
+                                               if !script::is_bolt2_compliant(&script, their_features) {
+                                                       return Err(ChannelError::Close(format!("Peer is signaling upfront_shutdown but has provided an unacceptable scriptpubkey format: {}", script)))
                                                }
+                                               Some(script.clone())
                                        }
                                },
                                // Peer is signaling upfront shutdown but don't opt-out with correct mechanism (a.k.a 0-length script). Peer looks buggy, we fail the channel
@@ -985,7 +973,7 @@ impl<Signer: Sign> Channel<Signer> {
                        feerate_per_kw: msg.feerate_per_kw,
                        channel_value_satoshis: msg.funding_satoshis,
                        counterparty_dust_limit_satoshis: msg.dust_limit_satoshis,
-                       holder_dust_limit_satoshis: MIN_DUST_LIMIT_SATOSHIS,
+                       holder_dust_limit_satoshis: MIN_CHAN_DUST_LIMIT_SATOSHIS,
                        counterparty_max_htlc_value_in_flight_msat: cmp::min(msg.max_htlc_value_in_flight_msat, msg.funding_satoshis * 1000),
                        counterparty_selected_channel_reserve_satoshis: Some(msg.channel_reserve_satoshis),
                        counterparty_htlc_minimum_msat: msg.htlc_minimum_msat,
@@ -1282,63 +1270,38 @@ impl<Signer: Sign> Channel<Signer> {
        }
 
        #[inline]
-       fn build_closing_transaction(&self, proposed_total_fee_satoshis: u64, skip_remote_output: bool) -> (Transaction, u64) {
-               let txins = {
-                       let mut ins: Vec<TxIn> = Vec::new();
-                       ins.push(TxIn {
-                               previous_output: self.funding_outpoint().into_bitcoin_outpoint(),
-                               script_sig: Script::new(),
-                               sequence: 0xffffffff,
-                               witness: Vec::new(),
-                       });
-                       ins
-               };
-
+       fn build_closing_transaction(&self, proposed_total_fee_satoshis: u64, skip_remote_output: bool) -> (ClosingTransaction, u64) {
                assert!(self.pending_inbound_htlcs.is_empty());
                assert!(self.pending_outbound_htlcs.is_empty());
                assert!(self.pending_update_fee.is_none());
-               let mut txouts: Vec<(TxOut, ())> = Vec::new();
 
                let mut total_fee_satoshis = proposed_total_fee_satoshis;
-               let value_to_self: i64 = (self.value_to_self_msat as i64) / 1000 - if self.is_outbound() { total_fee_satoshis as i64 } else { 0 };
-               let value_to_remote: i64 = ((self.channel_value_satoshis * 1000 - self.value_to_self_msat) as i64 / 1000) - if self.is_outbound() { 0 } else { total_fee_satoshis as i64 };
+               let mut value_to_holder: i64 = (self.value_to_self_msat as i64) / 1000 - if self.is_outbound() { total_fee_satoshis as i64 } else { 0 };
+               let mut value_to_counterparty: i64 = ((self.channel_value_satoshis * 1000 - self.value_to_self_msat) as i64 / 1000) - if self.is_outbound() { 0 } else { total_fee_satoshis as i64 };
 
-               if value_to_self < 0 {
+               if value_to_holder < 0 {
                        assert!(self.is_outbound());
-                       total_fee_satoshis += (-value_to_self) as u64;
-               } else if value_to_remote < 0 {
+                       total_fee_satoshis += (-value_to_holder) as u64;
+               } else if value_to_counterparty < 0 {
                        assert!(!self.is_outbound());
-                       total_fee_satoshis += (-value_to_remote) as u64;
+                       total_fee_satoshis += (-value_to_counterparty) as u64;
                }
 
-               if !skip_remote_output && value_to_remote as u64 > self.holder_dust_limit_satoshis {
-                       txouts.push((TxOut {
-                               script_pubkey: self.counterparty_shutdown_scriptpubkey.clone().unwrap(),
-                               value: value_to_remote as u64
-                       }, ()));
+               if skip_remote_output || value_to_counterparty as u64 <= self.holder_dust_limit_satoshis {
+                       value_to_counterparty = 0;
                }
 
-               assert!(self.shutdown_scriptpubkey.is_some());
-               if value_to_self as u64 > self.holder_dust_limit_satoshis {
-                       txouts.push((TxOut {
-                               script_pubkey: self.get_closing_scriptpubkey(),
-                               value: value_to_self as u64
-                       }, ()));
+               if value_to_holder as u64 <= self.holder_dust_limit_satoshis {
+                       value_to_holder = 0;
                }
 
-               transaction_utils::sort_outputs(&mut txouts, |_, _| { cmp::Ordering::Equal }); // Ordering doesnt matter if they used our pubkey...
-
-               let mut outputs: Vec<TxOut> = Vec::new();
-               for out in txouts.drain(..) {
-                       outputs.push(out.0);
-               }
+               assert!(self.shutdown_scriptpubkey.is_some());
+               let holder_shutdown_script = self.get_closing_scriptpubkey();
+               let counterparty_shutdown_script = self.counterparty_shutdown_scriptpubkey.clone().unwrap();
+               let funding_outpoint = self.funding_outpoint().into_bitcoin_outpoint();
 
-               (Transaction {
-                       version: 2,
-                       lock_time: 0,
-                       input: txins,
-                       output: outputs,
-               }, total_fee_satoshis)
+               let closing_transaction = ClosingTransaction::new(value_to_holder as u64, value_to_counterparty as u64, holder_shutdown_script, counterparty_shutdown_script, funding_outpoint);
+               (closing_transaction, total_fee_satoshis)
        }
 
        fn funding_outpoint(&self) -> OutPoint {
@@ -1654,11 +1617,11 @@ impl<Signer: Sign> Channel<Signer> {
                if msg.max_accepted_htlcs < config.peer_channel_config_limits.min_max_accepted_htlcs {
                        return Err(ChannelError::Close(format!("max_accepted_htlcs ({}) is less than the user specified limit ({})", msg.max_accepted_htlcs, config.peer_channel_config_limits.min_max_accepted_htlcs)));
                }
-               if msg.dust_limit_satoshis < MIN_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is less than the implementation limit ({})", msg.dust_limit_satoshis, MIN_DUST_LIMIT_SATOSHIS)));
+               if msg.dust_limit_satoshis < MIN_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is less than the implementation limit ({})", msg.dust_limit_satoshis, MIN_CHAN_DUST_LIMIT_SATOSHIS)));
                }
-               if msg.dust_limit_satoshis > MAX_DUST_LIMIT_SATOSHIS {
-                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is greater than the implementation limit ({})", msg.dust_limit_satoshis, MAX_DUST_LIMIT_SATOSHIS)));
+               if msg.dust_limit_satoshis > MAX_CHAN_DUST_LIMIT_SATOSHIS {
+                       return Err(ChannelError::Close(format!("dust_limit_satoshis ({}) is greater than the implementation limit ({})", msg.dust_limit_satoshis, MAX_CHAN_DUST_LIMIT_SATOSHIS)));
                }
                if msg.minimum_depth > config.peer_channel_config_limits.max_minimum_depth {
                        return Err(ChannelError::Close(format!("We consider the minimum depth to be unreasonably large. Expected minimum: ({}). Actual: ({})", config.peer_channel_config_limits.max_minimum_depth, msg.minimum_depth)));
@@ -1677,10 +1640,10 @@ impl<Signer: Sign> Channel<Signer> {
                                        if script.len() == 0 {
                                                None
                                        } else {
-                                               match ShutdownScript::try_from((script.clone(), their_features)) {
-                                                       Ok(shutdown_script) => Some(shutdown_script.into_inner()),
-                                                       Err(_) => return Err(ChannelError::Close(format!("Peer is signaling upfront_shutdown but has provided an unacceptable scriptpubkey format: {}", script))),
+                                               if !script::is_bolt2_compliant(&script, their_features) {
+                                                       return Err(ChannelError::Close(format!("Peer is signaling upfront_shutdown but has provided an unacceptable scriptpubkey format: {}", script)));
                                                }
+                                               Some(script.clone())
                                        }
                                },
                                // Peer is signaling upfront shutdown but don't opt-out with correct mechanism (a.k.a 0-length script). Peer looks buggy, we fail the channel
@@ -2204,7 +2167,7 @@ impl<Signer: Sign> Channel<Signer> {
                // We can't accept HTLCs sent after we've sent a shutdown.
                let local_sent_shutdown = (self.channel_state & (ChannelState::ChannelFunded as u32 | ChannelState::LocalShutdownSent as u32)) != (ChannelState::ChannelFunded as u32);
                if local_sent_shutdown {
-                       pending_forward_status = create_pending_htlc_status(self, pending_forward_status, 0x1000|20);
+                       pending_forward_status = create_pending_htlc_status(self, pending_forward_status, 0x4000|8);
                }
                // If the remote has sent a shutdown prior to adding this HTLC, then they are in violation of the spec.
                let remote_sent_shutdown = (self.channel_state & (ChannelState::ChannelFunded as u32 | ChannelState::RemoteShutdownSent as u32)) != (ChannelState::ChannelFunded as u32);
@@ -3086,13 +3049,10 @@ impl<Signer: Sign> Channel<Signer> {
        /// monitor update failure must *not* have been sent to the remote end, and must instead
        /// have been dropped. They will be regenerated when monitor_updating_restored is called.
        pub fn monitor_update_failed(&mut self, resend_raa: bool, resend_commitment: bool, mut pending_forwards: Vec<(PendingHTLCInfo, u64)>, mut pending_fails: Vec<(HTLCSource, PaymentHash, HTLCFailReason)>) {
-               assert_eq!(self.channel_state & ChannelState::MonitorUpdateFailed as u32, 0);
-               self.monitor_pending_revoke_and_ack = resend_raa;
-               self.monitor_pending_commitment_signed = resend_commitment;
-               assert!(self.monitor_pending_forwards.is_empty());
-               mem::swap(&mut pending_forwards, &mut self.monitor_pending_forwards);
-               assert!(self.monitor_pending_failures.is_empty());
-               mem::swap(&mut pending_fails, &mut self.monitor_pending_failures);
+               self.monitor_pending_revoke_and_ack |= resend_raa;
+               self.monitor_pending_commitment_signed |= resend_commitment;
+               self.monitor_pending_forwards.append(&mut pending_forwards);
+               self.monitor_pending_failures.append(&mut pending_fails);
                self.channel_state |= ChannelState::MonitorUpdateFailed as u32;
        }
 
@@ -3440,7 +3400,7 @@ impl<Signer: Sign> Channel<Signer> {
                                cmp::max(normal_feerate as u64 * tx_weight / 1000 + self.config.force_close_avoidance_max_fee_satoshis,
                                        proposed_max_feerate as u64 * tx_weight / 1000)
                        } else {
-                               u64::max_value()
+                               self.channel_value_satoshis - (self.value_to_self_msat + 999) / 1000
                        };
 
                self.closing_fee_limits = Some((proposed_total_fee_satoshis, proposed_max_total_fee_satoshis));
@@ -3533,17 +3493,16 @@ impl<Signer: Sign> Channel<Signer> {
                }
                assert_eq!(self.channel_state & ChannelState::ShutdownComplete as u32, 0);
 
-               let shutdown_scriptpubkey = match ShutdownScript::try_from((msg.scriptpubkey.clone(), their_features)) {
-                       Ok(script) => script.into_inner(),
-                       Err(_) => return Err(ChannelError::Close(format!("Got a nonstandard scriptpubkey ({}) from remote peer", msg.scriptpubkey.to_bytes().to_hex()))),
-               };
+               if !script::is_bolt2_compliant(&msg.scriptpubkey, their_features) {
+                       return Err(ChannelError::Close(format!("Got a nonstandard scriptpubkey ({}) from remote peer", msg.scriptpubkey.to_bytes().to_hex())));
+               }
 
                if self.counterparty_shutdown_scriptpubkey.is_some() {
-                       if Some(&shutdown_scriptpubkey) != self.counterparty_shutdown_scriptpubkey.as_ref() {
-                               return Err(ChannelError::Close(format!("Got shutdown request with a scriptpubkey ({}) which did not match their previous scriptpubkey.", shutdown_scriptpubkey.to_bytes().to_hex())));
+                       if Some(&msg.scriptpubkey) != self.counterparty_shutdown_scriptpubkey.as_ref() {
+                               return Err(ChannelError::Close(format!("Got shutdown request with a scriptpubkey ({}) which did not match their previous scriptpubkey.", msg.scriptpubkey.to_bytes().to_hex())));
                        }
                } else {
-                       self.counterparty_shutdown_scriptpubkey = Some(shutdown_scriptpubkey);
+                       self.counterparty_shutdown_scriptpubkey = Some(msg.scriptpubkey.clone());
                }
 
                // If we have any LocalAnnounced updates we'll probably just get back an update_fail_htlc
@@ -3606,10 +3565,8 @@ impl<Signer: Sign> Channel<Signer> {
                Ok((shutdown, monitor_update, dropped_outbound_htlcs))
        }
 
-       fn build_signed_closing_transaction(&self, tx: &mut Transaction, counterparty_sig: &Signature, sig: &Signature) {
-               if tx.input.len() != 1 { panic!("Tried to sign closing transaction that had input count != 1!"); }
-               if tx.input[0].witness.len() != 0 { panic!("Tried to re-sign closing transaction"); }
-               if tx.output.len() > 2 { panic!("Tried to sign bogus closing transaction"); }
+       fn build_signed_closing_transaction(&self, closing_tx: &ClosingTransaction, counterparty_sig: &Signature, sig: &Signature) -> Transaction {
+               let mut tx = closing_tx.trust().built_transaction().clone();
 
                tx.input[0].witness.push(Vec::new()); // First is the multisig dummy
 
@@ -3626,6 +3583,7 @@ impl<Signer: Sign> Channel<Signer> {
                tx.input[0].witness[2].push(SigHashType::All as u8);
 
                tx.input[0].witness.push(self.get_funding_redeemscript().into_bytes());
+               tx
        }
 
        pub fn closing_signed<F: Deref>(&mut self, fee_estimator: &F, msg: &msgs::ClosingSigned) -> Result<(Option<msgs::ClosingSigned>, Option<Transaction>), ChannelError>
@@ -3658,7 +3616,7 @@ impl<Signer: Sign> Channel<Signer> {
                if used_total_fee != msg.fee_satoshis {
                        return Err(ChannelError::Close(format!("Remote sent us a closing_signed with a fee other than the value they can claim. Fee in message: {}. Actual closing tx fee: {}", msg.fee_satoshis, used_total_fee)));
                }
-               let mut sighash = hash_to_message!(&bip143::SigHashCache::new(&closing_tx).signature_hash(0, &funding_redeemscript, self.channel_value_satoshis, SigHashType::All)[..]);
+               let sighash = closing_tx.trust().get_sighash_all(&funding_redeemscript, self.channel_value_satoshis);
 
                match self.secp_ctx.verify(&sighash, &msg.signature, &self.get_counterparty_pubkeys().funding_pubkey) {
                        Ok(_) => {},
@@ -3666,18 +3624,24 @@ impl<Signer: Sign> Channel<Signer> {
                                // The remote end may have decided to revoke their output due to inconsistent dust
                                // limits, so check for that case by re-checking the signature here.
                                closing_tx = self.build_closing_transaction(msg.fee_satoshis, true).0;
-                               sighash = hash_to_message!(&bip143::SigHashCache::new(&closing_tx).signature_hash(0, &funding_redeemscript, self.channel_value_satoshis, SigHashType::All)[..]);
+                               let sighash = closing_tx.trust().get_sighash_all(&funding_redeemscript, self.channel_value_satoshis);
                                secp_check!(self.secp_ctx.verify(&sighash, &msg.signature, self.counterparty_funding_pubkey()), "Invalid closing tx signature from peer".to_owned());
                        },
                };
 
+               for outp in closing_tx.trust().built_transaction().output.iter() {
+                       if !outp.script_pubkey.is_witness_program() && outp.value < MAX_STD_OUTPUT_DUST_LIMIT_SATOSHIS {
+                               return Err(ChannelError::Close("Remote sent us a closing_signed with a dust output. Always use segwit closing scripts!".to_owned()));
+                       }
+               }
+
                assert!(self.shutdown_scriptpubkey.is_some());
                if let Some((last_fee, sig)) = self.last_sent_closing_fee {
                        if last_fee == msg.fee_satoshis {
-                               self.build_signed_closing_transaction(&mut closing_tx, &msg.signature, &sig);
+                               let tx = self.build_signed_closing_transaction(&mut closing_tx, &msg.signature, &sig);
                                self.channel_state = ChannelState::ShutdownComplete as u32;
                                self.update_time_counter += 1;
-                               return Ok((None, Some(closing_tx)));
+                               return Ok((None, Some(tx)));
                        }
                }
 
@@ -3685,20 +3649,20 @@ impl<Signer: Sign> Channel<Signer> {
 
                macro_rules! propose_fee {
                        ($new_fee: expr) => {
-                               let (mut tx, used_fee) = if $new_fee == msg.fee_satoshis {
+                               let (closing_tx, used_fee) = if $new_fee == msg.fee_satoshis {
                                        (closing_tx, $new_fee)
                                } else {
                                        self.build_closing_transaction($new_fee, false)
                                };
 
                                let sig = self.holder_signer
-                                       .sign_closing_transaction(&tx, &self.secp_ctx)
+                                       .sign_closing_transaction(&closing_tx, &self.secp_ctx)
                                        .map_err(|_| ChannelError::Close("External signer refused to sign closing transaction".to_owned()))?;
 
                                let signed_tx = if $new_fee == msg.fee_satoshis {
                                        self.channel_state = ChannelState::ShutdownComplete as u32;
                                        self.update_time_counter += 1;
-                                       self.build_signed_closing_transaction(&mut tx, &msg.signature, &sig);
+                                       let tx = self.build_signed_closing_transaction(&closing_tx, &msg.signature, &sig);
                                        Some(tx)
                                } else { None };
 
@@ -3728,7 +3692,8 @@ impl<Signer: Sign> Channel<Signer> {
 
                        if !self.is_outbound() {
                                // They have to pay, so pick the highest fee in the overlapping range.
-                               debug_assert_eq!(our_max_fee, u64::max_value()); // We should never set an upper bound
+                               // We should never set an upper bound aside from their full balance
+                               debug_assert_eq!(our_max_fee, self.channel_value_satoshis - (self.value_to_self_msat + 999) / 1000);
                                propose_fee!(cmp::min(max_fee_satoshis, our_max_fee));
                        } else {
                                if msg.fee_satoshis < our_min_fee || msg.fee_satoshis > our_max_fee {
@@ -5551,7 +5516,7 @@ mod tests {
        use bitcoin::hashes::hex::FromHex;
        use hex;
        use ln::{PaymentPreimage, PaymentHash};
-       use ln::channelmanager::HTLCSource;
+       use ln::channelmanager::{HTLCSource, MppId};
        use ln::channel::{Channel,InboundHTLCOutput,OutboundHTLCOutput,InboundHTLCState,OutboundHTLCState,HTLCOutputInCommitment,HTLCCandidate,HTLCInitiator,TxCreationKeys};
        use ln::channel::MAX_FUNDING_SATOSHIS;
        use ln::features::InitFeatures;
@@ -5725,6 +5690,7 @@ mod tests {
                                path: Vec::new(),
                                session_priv: SecretKey::from_slice(&hex::decode("0fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").unwrap()[..]).unwrap(),
                                first_hop_htlc_msat: 548,
+                               mpp_id: MppId([42; 32]),
                        }
                });
 
index ef16af455adc6f02f4c8131d680f994b8af176ae..f154ab3dc79670186d46c1433462a7d256944e00 100644 (file)
@@ -43,7 +43,6 @@ use chain::transaction::{OutPoint, TransactionData};
 // Since this struct is returned in `list_channels` methods, expose it here in case users want to
 // construct one themselves.
 use ln::{PaymentHash, PaymentPreimage, PaymentSecret};
-pub use ln::channel::CounterpartyForwardingInfo;
 use ln::channel::{Channel, ChannelError, ChannelUpdateStatus, UpdateFulfillCommitFetch};
 use ln::features::{InitFeatures, NodeFeatures};
 use routing::router::{Route, RouteHop};
@@ -53,9 +52,9 @@ use ln::onion_utils;
 use ln::msgs::{ChannelMessageHandler, DecodeError, LightningError, OptionalField};
 use chain::keysinterface::{Sign, KeysInterface, KeysManager, InMemorySigner};
 use util::config::UserConfig;
-use util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
+use util::events::{EventHandler, EventsProvider, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
 use util::{byte_utils, events};
-use util::ser::{Readable, ReadableArgs, MaybeReadable, Writeable, Writer};
+use util::ser::{BigSize, FixedLengthReader, Readable, ReadableArgs, MaybeReadable, Writeable, Writer};
 use util::chacha20::{ChaCha20, ChaChaReader};
 use util::logger::{Logger, Level};
 use util::errors::APIError;
@@ -173,6 +172,22 @@ struct ClaimableHTLC {
        onion_payload: OnionPayload,
 }
 
+/// A payment identifier used to correlate an MPP payment's per-path HTLC sources internally.
+#[derive(Hash, Copy, Clone, PartialEq, Eq, Debug)]
+pub(crate) struct MppId(pub [u8; 32]);
+
+impl Writeable for MppId {
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               self.0.write(w)
+       }
+}
+
+impl Readable for MppId {
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let buf: [u8; 32] = Readable::read(r)?;
+               Ok(MppId(buf))
+       }
+}
 /// Tracks the inbound corresponding to an outbound HTLC
 #[derive(Clone, PartialEq)]
 pub(crate) enum HTLCSource {
@@ -183,6 +198,7 @@ pub(crate) enum HTLCSource {
                /// Technically we can recalculate this from the route, but we cache it here to avoid
                /// doing a double-pass on route when we get a failure back
                first_hop_htlc_msat: u64,
+               mpp_id: MppId,
        },
 }
 #[cfg(test)]
@@ -192,6 +208,7 @@ impl HTLCSource {
                        path: Vec::new(),
                        session_priv: SecretKey::from_slice(&[1; 32]).unwrap(),
                        first_hop_htlc_msat: 0,
+                       mpp_id: MppId([2; 32]),
                }
        }
 }
@@ -225,6 +242,7 @@ type ShutdownResult = (Option<(OutPoint, ChannelMonitorUpdate)>, Vec<(HTLCSource
 
 struct MsgHandleErrInternal {
        err: msgs::LightningError,
+       chan_id: Option<[u8; 32]>, // If Some a channel of ours has been closed
        shutdown_finish: Option<(ShutdownResult, Option<msgs::ChannelUpdate>)>,
 }
 impl MsgHandleErrInternal {
@@ -240,6 +258,7 @@ impl MsgHandleErrInternal {
                                        },
                                },
                        },
+                       chan_id: None,
                        shutdown_finish: None,
                }
        }
@@ -250,12 +269,13 @@ impl MsgHandleErrInternal {
                                err,
                                action: msgs::ErrorAction::IgnoreError,
                        },
+                       chan_id: None,
                        shutdown_finish: None,
                }
        }
        #[inline]
        fn from_no_close(err: msgs::LightningError) -> Self {
-               Self { err, shutdown_finish: None }
+               Self { err, chan_id: None, shutdown_finish: None }
        }
        #[inline]
        fn from_finish_shutdown(err: String, channel_id: [u8; 32], shutdown_res: ShutdownResult, channel_update: Option<msgs::ChannelUpdate>) -> Self {
@@ -269,6 +289,7 @@ impl MsgHandleErrInternal {
                                        },
                                },
                        },
+                       chan_id: Some(channel_id),
                        shutdown_finish: Some((shutdown_res, channel_update)),
                }
        }
@@ -303,6 +324,7 @@ impl MsgHandleErrInternal {
                                        },
                                },
                        },
+                       chan_id: None,
                        shutdown_finish: None,
                }
        }
@@ -467,14 +489,17 @@ pub struct ChannelManager<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref,
        /// The session_priv bytes of outbound payments which are pending resolution.
        /// The authoritative state of these HTLCs resides either within Channels or ChannelMonitors
        /// (if the channel has been force-closed), however we track them here to prevent duplicative
-       /// PaymentSent/PaymentFailed events. Specifically, in the case of a duplicative
+       /// PaymentSent/PaymentPathFailed events. Specifically, in the case of a duplicative
        /// update_fulfill_htlc message after a reconnect, we may "claim" a payment twice.
        /// Additionally, because ChannelMonitors are often not re-serialized after connecting block(s)
        /// which may generate a claim event, we may receive similar duplicate claim/fail MonitorEvents
        /// after reloading from disk while replaying blocks against ChannelMonitors.
        ///
+       /// Each payment has each of its MPP part's session_priv bytes in the HashSet of the map (even
+       /// payments over a single path).
+       ///
        /// Locked *after* channel_state.
-       pending_outbound_payments: Mutex<HashSet<[u8; 32]>>,
+       pending_outbound_payments: Mutex<HashMap<MppId, HashSet<[u8; 32]>>>,
 
        our_network_key: SecretKey,
        our_network_pubkey: PublicKey,
@@ -626,6 +651,19 @@ const CHECK_CLTV_EXPIRY_SANITY: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_GRA
 #[allow(dead_code)]
 const CHECK_CLTV_EXPIRY_SANITY_2: u32 = MIN_CLTV_EXPIRY_DELTA as u32 - LATENCY_GRACE_PERIOD_BLOCKS - 2*CLTV_CLAIM_BUFFER;
 
+/// Information needed for constructing an invoice route hint for this channel.
+#[derive(Clone, Debug, PartialEq)]
+pub struct CounterpartyForwardingInfo {
+       /// Base routing fee in millisatoshis.
+       pub fee_base_msat: u32,
+       /// Amount in millionths of a satoshi the channel will charge per transferred satoshi.
+       pub fee_proportional_millionths: u32,
+       /// The minimum difference in cltv_expiry between an ingoing HTLC and its outgoing counterpart,
+       /// such that the outgoing HTLC is forwardable to this counterparty. See `msgs::ChannelUpdate`'s
+       /// `cltv_expiry_delta` for more details.
+       pub cltv_expiry_delta: u16,
+}
+
 /// Channel parameters which apply to our counterparty. These are split out from [`ChannelDetails`]
 /// to better separate parameters.
 #[derive(Clone, Debug, PartialEq)]
@@ -780,12 +818,13 @@ macro_rules! handle_error {
        ($self: ident, $internal: expr, $counterparty_node_id: expr) => {
                match $internal {
                        Ok(msg) => Ok(msg),
-                       Err(MsgHandleErrInternal { err, shutdown_finish }) => {
+                       Err(MsgHandleErrInternal { err, chan_id, shutdown_finish }) => {
                                #[cfg(debug_assertions)]
                                {
                                        // In testing, ensure there are no deadlocks where the lock is already held upon
                                        // entering the macro.
                                        assert!($self.channel_state.try_lock().is_ok());
+                                       assert!($self.pending_events.try_lock().is_ok());
                                }
 
                                let mut msg_events = Vec::with_capacity(2);
@@ -797,6 +836,9 @@ macro_rules! handle_error {
                                                        msg: update
                                                });
                                        }
+                                       if let Some(channel_id) = chan_id {
+                                               $self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id,  reason: ClosureReason::ProcessingError { err: err.err.clone() } });
+                                       }
                                }
 
                                log_error!($self.logger, "{}", err.err);
@@ -1138,7 +1180,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                pending_msg_events: Vec::new(),
                        }),
                        pending_inbound_payments: Mutex::new(HashMap::new()),
-                       pending_outbound_payments: Mutex::new(HashSet::new()),
+                       pending_outbound_payments: Mutex::new(HashMap::new()),
 
                        our_network_key: keys_manager.get_node_secret(),
                        our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &keys_manager.get_node_secret()),
@@ -1330,6 +1372,12 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                msg: channel_update
                                                        });
                                                }
+                                               if let Ok(mut pending_events_lock) = self.pending_events.lock() {
+                                                       pending_events_lock.push(events::Event::ChannelClosed {
+                                                               channel_id: *channel_id,
+                                                               reason: ClosureReason::HolderForceClosed
+                                                       });
+                                               }
                                        }
                                        break Ok(());
                                },
@@ -1405,7 +1453,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                }
        }
 
-       fn force_close_channel_with_peer(&self, channel_id: &[u8; 32], peer_node_id: Option<&PublicKey>) -> Result<PublicKey, APIError> {
+       /// `peer_node_id` should be set when we receive a message from a peer, but not set when the
+       /// user closes, which will be re-exposed as the `ChannelClosed` reason.
+       fn force_close_channel_with_peer(&self, channel_id: &[u8; 32], peer_node_id: Option<&PublicKey>, peer_msg: Option<&String>) -> Result<PublicKey, APIError> {
                let mut chan = {
                        let mut channel_state_lock = self.channel_state.lock().unwrap();
                        let channel_state = &mut *channel_state_lock;
@@ -1418,6 +1468,14 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                if let Some(short_id) = chan.get().get_short_channel_id() {
                                        channel_state.short_to_id.remove(&short_id);
                                }
+                               let mut pending_events_lock = self.pending_events.lock().unwrap();
+                               if peer_node_id.is_some() {
+                                       if let Some(peer_msg) = peer_msg {
+                                               pending_events_lock.push(events::Event::ChannelClosed { channel_id: *channel_id, reason: ClosureReason::CounterpartyForceClosed { peer_msg: peer_msg.to_string() } });
+                                       }
+                               } else {
+                                       pending_events_lock.push(events::Event::ChannelClosed { channel_id: *channel_id, reason: ClosureReason::HolderForceClosed });
+                               }
                                chan.remove_entry().1
                        } else {
                                return Err(APIError::ChannelUnavailable{err: "No such channel".to_owned()});
@@ -1439,7 +1497,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        /// the chain and rejecting new HTLCs on the given channel. Fails if channel_id is unknown to the manager.
        pub fn force_close_channel(&self, channel_id: &[u8; 32]) -> Result<(), APIError> {
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               match self.force_close_channel_with_peer(channel_id, None) {
+               match self.force_close_channel_with_peer(channel_id, None, None) {
                        Ok(counterparty_node_id) => {
                                self.channel_state.lock().unwrap().pending_msg_events.push(
                                        events::MessageSendEvent::HandleError {
@@ -1820,7 +1878,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        }
 
        // Only public for testing, this should otherwise never be called direcly
-       pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, keysend_preimage: &Option<PaymentPreimage>) -> Result<(), APIError> {
+       pub(crate) fn send_payment_along_path(&self, path: &Vec<RouteHop>, payment_hash: &PaymentHash, payment_secret: &Option<PaymentSecret>, total_value: u64, cur_height: u32, mpp_id: MppId, keysend_preimage: &Option<PaymentPreimage>) -> Result<(), APIError> {
                log_trace!(self.logger, "Attempting to send payment for path with next hop {}", path.first().unwrap().short_channel_id);
                let prng_seed = self.keys_manager.get_secure_random_bytes();
                let session_priv_bytes = self.keys_manager.get_secure_random_bytes();
@@ -1835,7 +1893,9 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let onion_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, prng_seed, payment_hash);
 
                let _persistence_guard = PersistenceNotifierGuard::notify_on_drop(&self.total_consistency_lock, &self.persistence_notifier);
-               assert!(self.pending_outbound_payments.lock().unwrap().insert(session_priv_bytes));
+               let mut pending_outbounds = self.pending_outbound_payments.lock().unwrap();
+               let sessions = pending_outbounds.entry(mpp_id).or_insert(HashSet::new());
+               assert!(sessions.insert(session_priv_bytes));
 
                let err: Result<(), _> = loop {
                        let mut channel_lock = self.channel_state.lock().unwrap();
@@ -1857,6 +1917,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                path: path.clone(),
                                                session_priv: session_priv.clone(),
                                                first_hop_htlc_msat: htlc_msat,
+                                               mpp_id,
                                        }, onion_packet, &self.logger), channel_state, chan)
                                } {
                                        Some((update_add, commitment_signed, monitor_update)) => {
@@ -1956,6 +2017,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let mut total_value = 0;
                let our_node_id = self.get_our_node_id();
                let mut path_errs = Vec::with_capacity(route.paths.len());
+               let mpp_id = MppId(self.keys_manager.get_secure_random_bytes());
                'path_check: for path in route.paths.iter() {
                        if path.len() < 1 || path.len() > 20 {
                                path_errs.push(Err(APIError::RouteError{err: "Path didn't go anywhere/had bogus size"}));
@@ -1977,7 +2039,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                let cur_height = self.best_block.read().unwrap().height() + 1;
                let mut results = Vec::new();
                for path in route.paths.iter() {
-                       results.push(self.send_payment_along_path(&path, &payment_hash, payment_secret, total_value, cur_height, &keysend_preimage));
+                       results.push(self.send_payment_along_path(&path, &payment_hash, payment_secret, total_value, cur_height, mpp_id, &keysend_preimage));
                }
                let mut has_ok = false;
                let mut has_err = false;
@@ -2379,6 +2441,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                                        if let Some(short_id) = channel.get_short_channel_id() {
                                                                                                channel_state.short_to_id.remove(&short_id);
                                                                                        }
+                                                                                       // ChannelClosed event is generated by handle_error for us.
                                                                                        Err(MsgHandleErrInternal::from_finish_shutdown(msg, channel_id, channel.force_shutdown(true), self.get_channel_update_for_broadcast(&channel).ok()))
                                                                                },
                                                                                ChannelError::CloseDelayBroadcast(_) => { panic!("Wait is only generated on receipt of channel_reestablish, which is handled by try_chan_entry, we don't bother to support it here"); }
@@ -2812,22 +2875,29 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        self.fail_htlc_backwards_internal(channel_state,
                                                htlc_src, &payment_hash, HTLCFailReason::Reason { failure_code, data: onion_failure_data});
                                },
-                               HTLCSource::OutboundRoute { session_priv, .. } => {
-                                       if {
-                                               let mut session_priv_bytes = [0; 32];
-                                               session_priv_bytes.copy_from_slice(&session_priv[..]);
-                                               self.pending_outbound_payments.lock().unwrap().remove(&session_priv_bytes)
-                                       } {
-                                               self.pending_events.lock().unwrap().push(
-                                                       events::Event::PaymentFailed {
-                                                               payment_hash,
-                                                               rejected_by_dest: false,
-#[cfg(test)]
-                                                               error_code: None,
-#[cfg(test)]
-                                                               error_data: None,
+                               HTLCSource::OutboundRoute { session_priv, mpp_id, path, .. } => {
+                                       let mut session_priv_bytes = [0; 32];
+                                       session_priv_bytes.copy_from_slice(&session_priv[..]);
+                                       let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+                                       if let hash_map::Entry::Occupied(mut sessions) = outbounds.entry(mpp_id) {
+                                               if sessions.get_mut().remove(&session_priv_bytes) {
+                                                       self.pending_events.lock().unwrap().push(
+                                                               events::Event::PaymentPathFailed {
+                                                                       payment_hash,
+                                                                       rejected_by_dest: false,
+                                                                       network_update: None,
+                                                                       all_paths_failed: sessions.get().len() == 0,
+                                                                       path: path.clone(),
+                                                                       #[cfg(test)]
+                                                                       error_code: None,
+                                                                       #[cfg(test)]
+                                                                       error_data: None,
+                                                               }
+                                                       );
+                                                       if sessions.get().len() == 0 {
+                                                               sessions.remove();
                                                        }
-                                               )
+                                               }
                                        } else {
                                                log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
                                        }
@@ -2852,12 +2922,21 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                // from block_connected which may run during initialization prior to the chain_monitor
                // being fully configured. See the docs for `ChannelManagerReadArgs` for more.
                match source {
-                       HTLCSource::OutboundRoute { ref path, session_priv, .. } => {
-                               if {
-                                       let mut session_priv_bytes = [0; 32];
-                                       session_priv_bytes.copy_from_slice(&session_priv[..]);
-                                       !self.pending_outbound_payments.lock().unwrap().remove(&session_priv_bytes)
-                               } {
+                       HTLCSource::OutboundRoute { ref path, session_priv, mpp_id, .. } => {
+                               let mut session_priv_bytes = [0; 32];
+                               session_priv_bytes.copy_from_slice(&session_priv[..]);
+                               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+                               let mut all_paths_failed = false;
+                               if let hash_map::Entry::Occupied(mut sessions) = outbounds.entry(mpp_id) {
+                                       if !sessions.get_mut().remove(&session_priv_bytes) {
+                                               log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
+                                               return;
+                                       }
+                                       if sessions.get().len() == 0 {
+                                               all_paths_failed = true;
+                                               sessions.remove();
+                                       }
+                               } else {
                                        log_trace!(self.logger, "Received duplicative fail for HTLC with payment_hash {}", log_bytes!(payment_hash.0));
                                        return;
                                }
@@ -2866,23 +2945,19 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                match &onion_error {
                                        &HTLCFailReason::LightningError { ref err } => {
 #[cfg(test)]
-                                               let (channel_update, payment_retryable, onion_error_code, onion_error_data) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
+                                               let (network_update, payment_retryable, onion_error_code, onion_error_data) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
 #[cfg(not(test))]
-                                               let (channel_update, payment_retryable, _, _) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
+                                               let (network_update, payment_retryable, _, _) = onion_utils::process_onion_failure(&self.secp_ctx, &self.logger, &source, err.data.clone());
                                                // TODO: If we decided to blame ourselves (or one of our channels) in
                                                // process_onion_failure we should close that channel as it implies our
                                                // next-hop is needlessly blaming us!
-                                               if let Some(update) = channel_update {
-                                                       self.channel_state.lock().unwrap().pending_msg_events.push(
-                                                               events::MessageSendEvent::PaymentFailureNetworkUpdate {
-                                                                       update,
-                                                               }
-                                                       );
-                                               }
                                                self.pending_events.lock().unwrap().push(
-                                                       events::Event::PaymentFailed {
+                                                       events::Event::PaymentPathFailed {
                                                                payment_hash: payment_hash.clone(),
                                                                rejected_by_dest: !payment_retryable,
+                                                               network_update,
+                                                               all_paths_failed,
+                                                               path: path.clone(),
 #[cfg(test)]
                                                                error_code: onion_error_code,
 #[cfg(test)]
@@ -2897,16 +2972,19 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                        ref data,
                                                        .. } => {
                                                // we get a fail_malformed_htlc from the first hop
-                                               // TODO: We'd like to generate a PaymentFailureNetworkUpdate for temporary
+                                               // TODO: We'd like to generate a NetworkUpdate for temporary
                                                // failures here, but that would be insufficient as get_route
                                                // generally ignores its view of our own channels as we provide them via
                                                // ChannelDetails.
                                                // TODO: For non-temporary failures, we really should be closing the
                                                // channel here as we apparently can't relay through them anyway.
                                                self.pending_events.lock().unwrap().push(
-                                                       events::Event::PaymentFailed {
+                                                       events::Event::PaymentPathFailed {
                                                                payment_hash: payment_hash.clone(),
                                                                rejected_by_dest: path.len() == 1,
+                                                               network_update: None,
+                                                               all_paths_failed,
+                                                               path: path.clone(),
 #[cfg(test)]
                                                                error_code: Some(*failure_code),
 #[cfg(test)]
@@ -3103,17 +3181,18 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
 
        fn claim_funds_internal(&self, mut channel_state_lock: MutexGuard<ChannelHolder<Signer>>, source: HTLCSource, payment_preimage: PaymentPreimage, forwarded_htlc_value_msat: Option<u64>, from_onchain: bool) {
                match source {
-                       HTLCSource::OutboundRoute { session_priv, .. } => {
+                       HTLCSource::OutboundRoute { session_priv, mpp_id, .. } => {
                                mem::drop(channel_state_lock);
-                               if {
-                                       let mut session_priv_bytes = [0; 32];
-                                       session_priv_bytes.copy_from_slice(&session_priv[..]);
-                                       self.pending_outbound_payments.lock().unwrap().remove(&session_priv_bytes)
-                               } {
-                                       let mut pending_events = self.pending_events.lock().unwrap();
-                                       pending_events.push(events::Event::PaymentSent {
-                                               payment_preimage
-                                       });
+                               let mut session_priv_bytes = [0; 32];
+                               session_priv_bytes.copy_from_slice(&session_priv[..]);
+                               let mut outbounds = self.pending_outbound_payments.lock().unwrap();
+                               let found_payment = if let Some(mut sessions) = outbounds.remove(&mpp_id) {
+                                       sessions.remove(&session_priv_bytes)
+                               } else { false };
+                               if found_payment {
+                                       self.pending_events.lock().unwrap().push(
+                                               events::Event::PaymentSent { payment_preimage }
+                                       );
                                } else {
                                        log_trace!(self.logger, "Received duplicative fulfill for HTLC with payment_preimage {}", log_bytes!(payment_preimage.0));
                                }
@@ -3504,6 +3583,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        msg: update
                                });
                        }
+                       self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id: msg.channel_id,  reason: ClosureReason::CooperativeClosure });
                }
                Ok(())
        }
@@ -3528,33 +3608,34 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                }
 
                                let create_pending_htlc_status = |chan: &Channel<Signer>, pending_forward_info: PendingHTLCStatus, error_code: u16| {
-                                       // Ensure error_code has the UPDATE flag set, since by default we send a
-                                       // channel update along as part of failing the HTLC.
-                                       assert!((error_code & 0x1000) != 0);
                                        // If the update_add is completely bogus, the call will Err and we will close,
                                        // but if we've sent a shutdown and they haven't acknowledged it yet, we just
                                        // want to reject the new HTLC and fail it backwards instead of forwarding.
                                        match pending_forward_info {
                                                PendingHTLCStatus::Forward(PendingHTLCInfo { ref incoming_shared_secret, .. }) => {
-                                                       let reason = if let Ok(upd) = self.get_channel_update_for_unicast(chan) {
-                                                               onion_utils::build_first_hop_failure_packet(incoming_shared_secret, error_code, &{
-                                                                       let mut res = Vec::with_capacity(8 + 128);
-                                                                       // TODO: underspecified, follow https://github.com/lightningnetwork/lightning-rfc/issues/791
-                                                                       res.extend_from_slice(&byte_utils::be16_to_array(0));
-                                                                       res.extend_from_slice(&upd.encode_with_len()[..]);
-                                                                       res
-                                                               }[..])
+                                                       let reason = if (error_code & 0x1000) != 0 {
+                                                               if let Ok(upd) = self.get_channel_update_for_unicast(chan) {
+                                                                       onion_utils::build_first_hop_failure_packet(incoming_shared_secret, error_code, &{
+                                                                               let mut res = Vec::with_capacity(8 + 128);
+                                                                               // TODO: underspecified, follow https://github.com/lightningnetwork/lightning-rfc/issues/791
+                                                                               res.extend_from_slice(&byte_utils::be16_to_array(0));
+                                                                               res.extend_from_slice(&upd.encode_with_len()[..]);
+                                                                               res
+                                                                       }[..])
+                                                               } else {
+                                                                       // The only case where we'd be unable to
+                                                                       // successfully get a channel update is if the
+                                                                       // channel isn't in the fully-funded state yet,
+                                                                       // implying our counterparty is trying to route
+                                                                       // payments over the channel back to themselves
+                                                                       // (because no one else should know the short_id
+                                                                       // is a lightning channel yet). We should have
+                                                                       // no problem just calling this
+                                                                       // unknown_next_peer (0x4000|10).
+                                                                       onion_utils::build_first_hop_failure_packet(incoming_shared_secret, 0x4000|10, &[])
+                                                               }
                                                        } else {
-                                                               // The only case where we'd be unable to
-                                                               // successfully get a channel update is if the
-                                                               // channel isn't in the fully-funded state yet,
-                                                               // implying our counterparty is trying to route
-                                                               // payments over the channel back to themselves
-                                                               // (cause no one else should know the short_id
-                                                               // is a lightning channel yet). We should have
-                                                               // no problem just calling this
-                                                               // unknown_next_peer (0x4000|10).
-                                                               onion_utils::build_first_hop_failure_packet(incoming_shared_secret, 0x4000|10, &[])
+                                                               onion_utils::build_first_hop_failure_packet(incoming_shared_secret, error_code, &[])
                                                        };
                                                        let msg = msgs::UpdateFailHTLC {
                                                                channel_id: msg.channel_id,
@@ -3898,7 +3979,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                self.fail_htlc_backwards_internal(self.channel_state.lock().unwrap(), htlc_update.source, &htlc_update.payment_hash, HTLCFailReason::Reason { failure_code: 0x4000 | 8, data: Vec::new() });
                                        }
                                },
-                               MonitorEvent::CommitmentTxBroadcasted(funding_outpoint) => {
+                               MonitorEvent::CommitmentTxConfirmed(funding_outpoint) => {
                                        let mut channel_lock = self.channel_state.lock().unwrap();
                                        let channel_state = &mut *channel_lock;
                                        let by_id = &mut channel_state.by_id;
@@ -3914,6 +3995,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                msg: update
                                                        });
                                                }
+                                               self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id: chan.channel_id(),  reason: ClosureReason::CommitmentTxConfirmed });
                                                pending_msg_events.push(events::MessageSendEvent::HandleError {
                                                        node_id: chan.get_counterparty_node_id(),
                                                        action: msgs::ErrorAction::SendErrorMessage {
@@ -3975,6 +4057,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                        Err(e) => {
                                                let (close_channel, res) = convert_chan_err!(self, e, short_to_id, chan, channel_id);
                                                handle_errors.push((chan.get_counterparty_node_id(), Err(res)));
+                                               // ChannelClosed event is generated by handle_error for us
                                                !close_channel
                                        }
                                }
@@ -4028,6 +4111,13 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
                                                                });
                                                        }
 
+                                                       if let Ok(mut pending_events_lock) = self.pending_events.lock() {
+                                                               pending_events_lock.push(events::Event::ChannelClosed {
+                                                                       channel_id: *channel_id,
+                                                                       reason: ClosureReason::CooperativeClosure
+                                                               });
+                                                       }
+
                                                        log_info!(self.logger, "Broadcasting {}", log_tx!(tx));
                                                        self.tx_broadcaster.broadcast_transaction(&tx);
                                                        false
@@ -4176,7 +4266,7 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> ChannelMana
        #[cfg(any(test, feature = "fuzztarget", feature = "_test_utils"))]
        pub fn get_and_clear_pending_events(&self) -> Vec<events::Event> {
                let events = core::cell::RefCell::new(Vec::new());
-               let event_handler = |event| events.borrow_mut().push(event);
+               let event_handler = |event: &events::Event| events.borrow_mut().push(event.clone());
                self.process_pending_events(&event_handler);
                events.into_inner()
        }
@@ -4253,7 +4343,7 @@ where
                        }
 
                        for event in pending_events.drain(..) {
-                               handler.handle_event(event);
+                               handler.handle_event(&event);
                        }
 
                        result
@@ -4448,6 +4538,7 @@ where
                                                        msg: update
                                                });
                                        }
+                                       self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id: channel.channel_id(),  reason: ClosureReason::CommitmentTxConfirmed });
                                        pending_msg_events.push(events::MessageSendEvent::HandleError {
                                                node_id: channel.get_counterparty_node_id(),
                                                action: msgs::ErrorAction::SendErrorMessage { msg: e },
@@ -4638,6 +4729,7 @@ impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                                                                msg: update
                                                        });
                                                }
+                                               self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id: chan.channel_id(),  reason: ClosureReason::DisconnectedPeer });
                                                false
                                        } else {
                                                true
@@ -4652,6 +4744,7 @@ impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                                                        if let Some(short_id) = chan.get_short_channel_id() {
                                                                short_to_id.remove(&short_id);
                                                        }
+                                                       self.pending_events.lock().unwrap().push(events::Event::ChannelClosed { channel_id: chan.channel_id(),  reason: ClosureReason::DisconnectedPeer });
                                                        return false;
                                                } else {
                                                        no_channels_remain = false;
@@ -4678,7 +4771,6 @@ impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                                        &events::MessageSendEvent::BroadcastChannelUpdate { .. } => true,
                                        &events::MessageSendEvent::SendChannelUpdate { ref node_id, .. } => node_id != counterparty_node_id,
                                        &events::MessageSendEvent::HandleError { ref node_id, .. } => node_id != counterparty_node_id,
-                                       &events::MessageSendEvent::PaymentFailureNetworkUpdate { .. } => true,
                                        &events::MessageSendEvent::SendChannelRangeQuery { .. } => false,
                                        &events::MessageSendEvent::SendShortIdsQuery { .. } => false,
                                        &events::MessageSendEvent::SendReplyChannelRange { .. } => false,
@@ -4743,12 +4835,12 @@ impl<Signer: Sign, M: Deref , T: Deref , K: Deref , F: Deref , L: Deref >
                        for chan in self.list_channels() {
                                if chan.counterparty.node_id == *counterparty_node_id {
                                        // Untrusted messages from peer, we throw away the error if id points to a non-existent channel
-                                       let _ = self.force_close_channel_with_peer(&chan.channel_id, Some(counterparty_node_id));
+                                       let _ = self.force_close_channel_with_peer(&chan.channel_id, Some(counterparty_node_id), Some(&msg.data));
                                }
                        }
                } else {
                        // Untrusted messages from peer, we throw away the error if id points to a non-existent channel
-                       let _ = self.force_close_channel_with_peer(&msg.channel_id, Some(counterparty_node_id));
+                       let _ = self.force_close_channel_with_peer(&msg.channel_id, Some(counterparty_node_id), Some(&msg.data));
                }
        }
 }
@@ -4850,10 +4942,74 @@ impl_writeable_tlv_based!(PendingHTLCInfo, {
        (8, outgoing_cltv_value, required)
 });
 
-impl_writeable_tlv_based_enum!(HTLCFailureMsg, ;
-       (0, Relay),
-       (1, Malformed),
-);
+
+impl Writeable for HTLCFailureMsg {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), io::Error> {
+               match self {
+                       HTLCFailureMsg::Relay(msgs::UpdateFailHTLC { channel_id, htlc_id, reason }) => {
+                               0u8.write(writer)?;
+                               channel_id.write(writer)?;
+                               htlc_id.write(writer)?;
+                               reason.write(writer)?;
+                       },
+                       HTLCFailureMsg::Malformed(msgs::UpdateFailMalformedHTLC {
+                               channel_id, htlc_id, sha256_of_onion, failure_code
+                       }) => {
+                               1u8.write(writer)?;
+                               channel_id.write(writer)?;
+                               htlc_id.write(writer)?;
+                               sha256_of_onion.write(writer)?;
+                               failure_code.write(writer)?;
+                       },
+               }
+               Ok(())
+       }
+}
+
+impl Readable for HTLCFailureMsg {
+       fn read<R: Read>(reader: &mut R) -> Result<Self, DecodeError> {
+               let id: u8 = Readable::read(reader)?;
+               match id {
+                       0 => {
+                               Ok(HTLCFailureMsg::Relay(msgs::UpdateFailHTLC {
+                                       channel_id: Readable::read(reader)?,
+                                       htlc_id: Readable::read(reader)?,
+                                       reason: Readable::read(reader)?,
+                               }))
+                       },
+                       1 => {
+                               Ok(HTLCFailureMsg::Malformed(msgs::UpdateFailMalformedHTLC {
+                                       channel_id: Readable::read(reader)?,
+                                       htlc_id: Readable::read(reader)?,
+                                       sha256_of_onion: Readable::read(reader)?,
+                                       failure_code: Readable::read(reader)?,
+                               }))
+                       },
+                       // In versions prior to 0.0.101, HTLCFailureMsg objects were written with type 0 or 1 but
+                       // weren't length-prefixed and thus didn't support reading the TLV stream suffix of the network
+                       // messages contained in the variants.
+                       // In version 0.0.101, support for reading the variants with these types was added, and
+                       // we should migrate to writing these variants when UpdateFailHTLC or
+                       // UpdateFailMalformedHTLC get TLV fields.
+                       2 => {
+                               let length: BigSize = Readable::read(reader)?;
+                               let mut s = FixedLengthReader::new(reader, length.0);
+                               let res = Readable::read(&mut s)?;
+                               s.eat_remaining()?; // Return ShortRead if there's actually not enough bytes
+                               Ok(HTLCFailureMsg::Relay(res))
+                       },
+                       3 => {
+                               let length: BigSize = Readable::read(reader)?;
+                               let mut s = FixedLengthReader::new(reader, length.0);
+                               let res = Readable::read(&mut s)?;
+                               s.eat_remaining()?; // Return ShortRead if there's actually not enough bytes
+                               Ok(HTLCFailureMsg::Malformed(res))
+                       },
+                       _ => Err(DecodeError::UnknownRequiredFeature),
+               }
+       }
+}
+
 impl_writeable_tlv_based_enum!(PendingHTLCStatus, ;
        (0, Forward),
        (1, Fail),
@@ -4924,14 +5080,60 @@ impl Readable for ClaimableHTLC {
        }
 }
 
-impl_writeable_tlv_based_enum!(HTLCSource,
-       (0, OutboundRoute) => {
-               (0, session_priv, required),
-               (2, first_hop_htlc_msat, required),
-               (4, path, vec_type),
-       }, ;
-       (1, PreviousHopData)
-);
+impl Readable for HTLCSource {
+       fn read<R: Read>(reader: &mut R) -> Result<Self, DecodeError> {
+               let id: u8 = Readable::read(reader)?;
+               match id {
+                       0 => {
+                               let mut session_priv: ::util::ser::OptionDeserWrapper<SecretKey> = ::util::ser::OptionDeserWrapper(None);
+                               let mut first_hop_htlc_msat: u64 = 0;
+                               let mut path = Some(Vec::new());
+                               let mut mpp_id = None;
+                               read_tlv_fields!(reader, {
+                                       (0, session_priv, required),
+                                       (1, mpp_id, option),
+                                       (2, first_hop_htlc_msat, required),
+                                       (4, path, vec_type),
+                               });
+                               if mpp_id.is_none() {
+                                       // For backwards compat, if there was no mpp_id written, use the session_priv bytes
+                                       // instead.
+                                       mpp_id = Some(MppId(*session_priv.0.unwrap().as_ref()));
+                               }
+                               Ok(HTLCSource::OutboundRoute {
+                                       session_priv: session_priv.0.unwrap(),
+                                       first_hop_htlc_msat: first_hop_htlc_msat,
+                                       path: path.unwrap(),
+                                       mpp_id: mpp_id.unwrap(),
+                               })
+                       }
+                       1 => Ok(HTLCSource::PreviousHopData(Readable::read(reader)?)),
+                       _ => Err(DecodeError::UnknownRequiredFeature),
+               }
+       }
+}
+
+impl Writeable for HTLCSource {
+       fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ::io::Error> {
+               match self {
+                       HTLCSource::OutboundRoute { ref session_priv, ref first_hop_htlc_msat, ref path, mpp_id } => {
+                               0u8.write(writer)?;
+                               let mpp_id_opt = Some(mpp_id);
+                               write_tlv_fields!(writer, {
+                                       (0, session_priv, required),
+                                       (1, mpp_id_opt, option),
+                                       (2, first_hop_htlc_msat, required),
+                                       (4, path, vec_type),
+                                });
+                       }
+                       HTLCSource::PreviousHopData(ref field) => {
+                               1u8.write(writer)?;
+                               field.write(writer)?;
+                       }
+               }
+               Ok(())
+       }
+}
 
 impl_writeable_tlv_based_enum!(HTLCFailReason,
        (0, LightningError) => {
@@ -5052,12 +5254,21 @@ impl<Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref> Writeable f
                }
 
                let pending_outbound_payments = self.pending_outbound_payments.lock().unwrap();
-               (pending_outbound_payments.len() as u64).write(writer)?;
-               for session_priv in pending_outbound_payments.iter() {
-                       session_priv.write(writer)?;
+               // For backwards compat, write the session privs and their total length.
+               let mut num_pending_outbounds_compat: u64 = 0;
+               for (_, outbounds) in pending_outbound_payments.iter() {
+                       num_pending_outbounds_compat += outbounds.len() as u64;
+               }
+               num_pending_outbounds_compat.write(writer)?;
+               for (_, outbounds) in pending_outbound_payments.iter() {
+                       for outbound in outbounds.iter() {
+                               outbound.write(writer)?;
+                       }
                }
 
-               write_tlv_fields!(writer, {});
+               write_tlv_fields!(writer, {
+                       (1, pending_outbound_payments, required),
+               });
 
                Ok(())
        }
@@ -5194,6 +5405,7 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                let mut funding_txo_set = HashSet::with_capacity(cmp::min(channel_count as usize, 128));
                let mut by_id = HashMap::with_capacity(cmp::min(channel_count as usize, 128));
                let mut short_to_id = HashMap::with_capacity(cmp::min(channel_count as usize, 128));
+               let mut channel_closures = Vec::new();
                for _ in 0..channel_count {
                        let mut channel: Channel<Signer> = Channel::read(reader, &args.keys_manager)?;
                        let funding_txo = channel.get_funding_txo().ok_or(DecodeError::InvalidValue)?;
@@ -5224,6 +5436,10 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                        let (_, mut new_failed_htlcs) = channel.force_shutdown(true);
                                        failed_htlcs.append(&mut new_failed_htlcs);
                                        monitor.broadcast_latest_holder_commitment_txn(&args.tx_broadcaster, &args.logger);
+                                       channel_closures.push(events::Event::ChannelClosed {
+                                               channel_id: channel.channel_id(),
+                                               reason: ClosureReason::OutdatedChannelManager
+                                       });
                                } else {
                                        if let Some(short_channel_id) = channel.get_short_channel_id() {
                                                short_to_id.insert(short_channel_id, channel.channel_id());
@@ -5289,6 +5505,16 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                None => continue,
                        }
                }
+               if forward_htlcs_count > 0 {
+                       // If we have pending HTLCs to forward, assume we either dropped a
+                       // `PendingHTLCsForwardable` or the user received it but never processed it as they
+                       // shut down before the timer hit. Either way, set the time_forwardable to a small
+                       // constant as enough time has likely passed that we should simply handle the forwards
+                       // now, or at least after the user gets a chance to reconnect to our peers.
+                       pending_events_read.push(events::Event::PendingHTLCsForwardable {
+                               time_forwardable: Duration::from_secs(2),
+                       });
+               }
 
                let background_event_count: u64 = Readable::read(reader)?;
                let mut pending_background_events_read: Vec<BackgroundEvent> = Vec::with_capacity(cmp::min(background_event_count as usize, MAX_ALLOC_SIZE/mem::size_of::<BackgroundEvent>()));
@@ -5310,19 +5536,31 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                        }
                }
 
-               let pending_outbound_payments_count: u64 = Readable::read(reader)?;
-               let mut pending_outbound_payments: HashSet<[u8; 32]> = HashSet::with_capacity(cmp::min(pending_outbound_payments_count as usize, MAX_ALLOC_SIZE/32));
-               for _ in 0..pending_outbound_payments_count {
-                       if !pending_outbound_payments.insert(Readable::read(reader)?) {
-                               return Err(DecodeError::InvalidValue);
-                       }
+               let pending_outbound_payments_count_compat: u64 = Readable::read(reader)?;
+               let mut pending_outbound_payments_compat: HashMap<MppId, HashSet<[u8; 32]>> =
+                       HashMap::with_capacity(cmp::min(pending_outbound_payments_count_compat as usize, MAX_ALLOC_SIZE/32));
+               for _ in 0..pending_outbound_payments_count_compat {
+                       let session_priv = Readable::read(reader)?;
+                       if pending_outbound_payments_compat.insert(MppId(session_priv), [session_priv].iter().cloned().collect()).is_some() {
+                               return Err(DecodeError::InvalidValue)
+                       };
                }
 
-               read_tlv_fields!(reader, {});
+               let mut pending_outbound_payments = None;
+               read_tlv_fields!(reader, {
+                       (1, pending_outbound_payments, option),
+               });
+               if pending_outbound_payments.is_none() {
+                       pending_outbound_payments = Some(pending_outbound_payments_compat);
+               }
 
                let mut secp_ctx = Secp256k1::new();
                secp_ctx.seeded_randomize(&args.keys_manager.get_secure_random_bytes());
 
+               if !channel_closures.is_empty() {
+                       pending_events_read.append(&mut channel_closures);
+               }
+
                let channel_manager = ChannelManager {
                        genesis_hash,
                        fee_estimator: args.fee_estimator,
@@ -5339,7 +5577,7 @@ impl<'a, Signer: Sign, M: Deref, T: Deref, K: Deref, F: Deref, L: Deref>
                                pending_msg_events: Vec::new(),
                        }),
                        pending_inbound_payments: Mutex::new(pending_inbound_payments),
-                       pending_outbound_payments: Mutex::new(pending_outbound_payments),
+                       pending_outbound_payments: Mutex::new(pending_outbound_payments.unwrap()),
 
                        our_network_key: args.keys_manager.get_node_secret(),
                        our_network_pubkey: PublicKey::from_secret_key(&secp_ctx, &args.keys_manager.get_node_secret()),
@@ -5377,7 +5615,7 @@ mod tests {
        use bitcoin::hashes::sha256::Hash as Sha256;
        use core::time::Duration;
        use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-       use ln::channelmanager::PaymentSendFailure;
+       use ln::channelmanager::{MppId, PaymentSendFailure};
        use ln::features::{InitFeatures, InvoiceFeatures};
        use ln::functional_test_utils::*;
        use ln::msgs;
@@ -5526,12 +5764,13 @@ mod tests {
 
                // First, send a partial MPP payment.
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
                let (payment_preimage, our_payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[1]);
+               let mpp_id = MppId([42; 32]);
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
-               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, &None).unwrap();
+               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, mpp_id, &None).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -5561,7 +5800,7 @@ mod tests {
                expect_payment_failed!(nodes[0], our_payment_hash, true);
 
                // Send the second half of the original MPP payment.
-               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, &None).unwrap();
+               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200_000, cur_height, mpp_id, &None).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -5599,7 +5838,8 @@ mod tests {
                nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_third_raa);
                check_added_monitors!(nodes[0], 1);
 
-               // There's an existing bug that generates a PaymentSent event for each MPP path, so handle that here.
+               // Note that successful MPP payments will generate 1 event upon the first path's success. No
+               // further events will be generated for subsequence path successes.
                let events = nodes[0].node.get_and_clear_pending_events();
                match events[0] {
                        Event::PaymentSent { payment_preimage: ref preimage } => {
@@ -5607,12 +5847,6 @@ mod tests {
                        },
                        _ => panic!("Unexpected event"),
                }
-               match events[1] {
-                       Event::PaymentSent { payment_preimage: ref preimage } => {
-                               assert_eq!(payment_preimage, *preimage);
-                       },
-                       _ => panic!("Unexpected event"),
-               }
        }
 
        #[test]
@@ -5633,7 +5867,7 @@ mod tests {
                let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &expected_route, 100_000);
 
                // Next, attempt a keysend payment and make sure it fails.
-               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -5661,7 +5895,7 @@ mod tests {
 
                // To start (2), send a keysend payment but don't claim it.
                let payment_preimage = PaymentPreimage([42; 32]);
-               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100_000, TEST_FINAL_CLTV, &logger).unwrap();
                let payment_hash = nodes[0].node.send_spontaneous_payment(&route, Some(payment_preimage)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -5713,9 +5947,9 @@ mod tests {
                nodes[1].node.peer_connected(&payer_pubkey, &msgs::Init { features: InitFeatures::known() });
 
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], InitFeatures::known(), InitFeatures::known());
-               let network_graph = nodes[0].net_graph_msg_handler.network_graph.read().unwrap();
+               let network_graph = &nodes[0].net_graph_msg_handler.network_graph;
                let first_hops = nodes[0].node.list_usable_channels();
-               let route = get_keysend_route(&payer_pubkey, &network_graph, &payee_pubkey,
+               let route = get_keysend_route(&payer_pubkey, network_graph, &payee_pubkey,
                                   Some(&first_hops.iter().collect::<Vec<_>>()), &vec![], 10000, 40,
                                   nodes[0].logger).unwrap();
 
@@ -5749,9 +5983,9 @@ mod tests {
                nodes[1].node.peer_connected(&payer_pubkey, &msgs::Init { features: InitFeatures::known() });
 
                let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], InitFeatures::known(), InitFeatures::known());
-               let network_graph = nodes[0].net_graph_msg_handler.network_graph.read().unwrap();
+               let network_graph = &nodes[0].net_graph_msg_handler.network_graph;
                let first_hops = nodes[0].node.list_usable_channels();
-               let route = get_keysend_route(&payer_pubkey, &network_graph, &payee_pubkey,
+               let route = get_keysend_route(&payer_pubkey, network_graph, &payee_pubkey,
                                   Some(&first_hops.iter().collect::<Vec<_>>()), &vec![], 10000, 40,
                                   nodes[0].logger).unwrap();
 
@@ -5788,7 +6022,7 @@ mod tests {
                // Marshall an MPP route.
                let (_, payment_hash, _) = get_payment_preimage_hash!(&nodes[3]);
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
                let path = route.paths[0].clone();
                route.paths.push(path);
                route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
index d1f6b89db4f87e968e3624289331c00c476875fe..8f251d8478c7048f33cfb003725f4c94b388a209 100644 (file)
@@ -392,7 +392,6 @@ impl InitFeatures {
        /// Writes all features present up to, and including, 13.
        pub(crate) fn write_up_to_13<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                let len = cmp::min(2, self.flags.len());
-               w.size_hint(len + 2);
                (len as u16).write(w)?;
                for i in (0..len).rev() {
                        if i == 0 {
@@ -584,12 +583,6 @@ impl<T: sealed::Context> Features<T> {
                        (byte & unknown_features) != 0
                })
        }
-
-       /// The number of bytes required to represent the feature flags present. This does not include
-       /// the length bytes which are included in the serialized form.
-       pub(crate) fn byte_count(&self) -> usize {
-               self.flags.len()
-       }
 }
 
 impl<T: sealed::DataLossProtect> Features<T> {
@@ -702,7 +695,6 @@ impl<T: sealed::ShutdownAnySegwit> Features<T> {
 
 impl<T: sealed::Context> Writeable for Features<T> {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(self.flags.len() + 2);
                (self.flags.len() as u16).write(w)?;
                for f in self.flags.iter().rev() { // Swap back to big-endian
                        f.write(w)?;
@@ -835,7 +827,7 @@ mod tests {
        #[test]
        fn convert_to_context_with_unknown_flags() {
                // Ensure the `from` context has fewer known feature bytes than the `to` context.
-               assert!(InvoiceFeatures::known().byte_count() < NodeFeatures::known().byte_count());
+               assert!(InvoiceFeatures::known().flags.len() < NodeFeatures::known().flags.len());
                let invoice_features = InvoiceFeatures::known().set_unknown_feature_optional();
                assert!(invoice_features.supports_unknown_bits());
                let node_features: NodeFeatures = invoice_features.to_context();
index 82c5d2901e2c15ddedf1ca35548492207f392858..9fbbcfff39c9701306f0c06e0dcc47c0fdd7728b 100644 (file)
@@ -239,12 +239,12 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                        // Check that if we serialize the Router, we can deserialize it again.
                        {
                                let mut w = test_utils::TestVecWriter(Vec::new());
-                               let network_graph_ser = self.net_graph_msg_handler.network_graph.read().unwrap();
+                               let network_graph_ser = &self.net_graph_msg_handler.network_graph;
                                network_graph_ser.write(&mut w).unwrap();
                                let network_graph_deser = <NetworkGraph>::read(&mut io::Cursor::new(&w.0)).unwrap();
-                               assert!(network_graph_deser == *self.net_graph_msg_handler.network_graph.read().unwrap());
-                               let net_graph_msg_handler = NetGraphMsgHandler::from_net_graph(
-                                       Some(self.chain_source), self.logger, network_graph_deser
+                               assert!(network_graph_deser == self.net_graph_msg_handler.network_graph);
+                               let net_graph_msg_handler = NetGraphMsgHandler::new(
+                                       network_graph_deser, Some(self.chain_source), self.logger
                                );
                                let mut chan_progress = 0;
                                loop {
@@ -743,16 +743,16 @@ macro_rules! get_closing_signed_broadcast {
 #[macro_export]
 macro_rules! check_closed_broadcast {
        ($node: expr, $with_error_msg: expr) => {{
-               let events = $node.node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), if $with_error_msg { 2 } else { 1 });
-               match events[0] {
+               let msg_events = $node.node.get_and_clear_pending_msg_events();
+               assert_eq!(msg_events.len(), if $with_error_msg { 2 } else { 1 });
+               match msg_events[0] {
                        MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                                assert_eq!(msg.contents.flags & 2, 2);
                        },
                        _ => panic!("Unexpected event"),
                }
                if $with_error_msg {
-                       match events[1] {
+                       match msg_events[1] {
                                MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { ref msg }, node_id: _ } => {
                                        // TODO: Check node_id
                                        Some(msg.clone())
@@ -763,6 +763,24 @@ macro_rules! check_closed_broadcast {
        }}
 }
 
+/// Check that a channel's closing channel event has been issued
+#[macro_export]
+macro_rules! check_closed_event {
+       ($node: expr, $events: expr, $reason: expr) => {{
+               let events = $node.node.get_and_clear_pending_events();
+               assert_eq!(events.len(), $events);
+               let expected_reason = $reason;
+               for event in events {
+                       match event {
+                               Event::ChannelClosed { ref reason, .. } => {
+                                       assert_eq!(*reason, expected_reason);
+                               },
+                               _ => panic!("Unexpected event"),
+                       }
+               }
+       }}
+}
+
 pub fn close_channel<'a, 'b, 'c>(outbound_node: &Node<'a, 'b, 'c>, inbound_node: &Node<'a, 'b, 'c>, channel_id: &[u8; 32], funding_tx: Transaction, close_inbound_first: bool) -> (msgs::ChannelUpdate, msgs::ChannelUpdate, Transaction) {
        let (node_a, broadcaster_a, struct_a) = if close_inbound_first { (&inbound_node.node, &inbound_node.tx_broadcaster, inbound_node) } else { (&outbound_node.node, &outbound_node.tx_broadcaster, outbound_node) };
        let (node_b, broadcaster_b, struct_b) = if close_inbound_first { (&outbound_node.node, &outbound_node.tx_broadcaster, outbound_node) } else { (&inbound_node.node, &inbound_node.tx_broadcaster, inbound_node) };
@@ -962,7 +980,7 @@ macro_rules! get_route_and_payment_hash {
                let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!($recv_node);
                let net_graph_msg_handler = &$send_node.net_graph_msg_handler;
                let route = get_route(&$send_node.node.get_our_node_id(),
-                       &net_graph_msg_handler.network_graph.read().unwrap(),
+                       &net_graph_msg_handler.network_graph,
                        &$recv_node.node.get_our_node_id(), None, None, &Vec::new(), $recv_value, TEST_FINAL_CLTV, $send_node.logger).unwrap();
                (route, payment_hash, payment_preimage, payment_secret)
        }}
@@ -986,6 +1004,20 @@ macro_rules! expect_pending_htlcs_forwardable {
        }}
 }
 
+#[cfg(test)]
+macro_rules! expect_pending_htlcs_forwardable_from_events {
+       ($node: expr, $events: expr, $ignore: expr) => {{
+               assert_eq!($events.len(), 1);
+               match $events[0] {
+                       Event::PendingHTLCsForwardable { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+               if $ignore {
+                       $node.node.process_pending_htlc_forwards();
+               }
+       }}
+}
+
 #[cfg(any(test, feature = "unstable"))]
 macro_rules! expect_payment_received {
        ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
@@ -1036,22 +1068,27 @@ macro_rules! expect_payment_forwarded {
 }
 
 #[cfg(test)]
-macro_rules! expect_payment_failure_chan_update {
-       ($node: expr, $scid: expr, $chan_closed: expr) => {
-               let events = $node.node.get_and_clear_pending_msg_events();
+macro_rules! expect_payment_failed_with_update {
+       ($node: expr, $expected_payment_hash: expr, $rejected_by_dest: expr, $scid: expr, $chan_closed: expr) => {
+               let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       MessageSendEvent::PaymentFailureNetworkUpdate { ref update } => {
-                               match update {
-                                       &HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg } if !$chan_closed => {
+                       Event::PaymentPathFailed { ref payment_hash, rejected_by_dest, ref network_update, ref error_code, ref error_data, .. } => {
+                               assert_eq!(*payment_hash, $expected_payment_hash, "unexpected payment_hash");
+                               assert_eq!(rejected_by_dest, $rejected_by_dest, "unexpected rejected_by_dest value");
+                               assert!(error_code.is_some(), "expected error_code.is_some() = true");
+                               assert!(error_data.is_some(), "expected error_data.is_some() = true");
+                               match network_update {
+                                       &Some(NetworkUpdate::ChannelUpdateMessage { ref msg }) if !$chan_closed => {
                                                assert_eq!(msg.contents.short_channel_id, $scid);
                                                assert_eq!(msg.contents.flags & 2, 0);
                                        },
-                                       &HTLCFailChannelUpdate::ChannelClosed { short_channel_id, is_permanent } if $chan_closed => {
+                                       &Some(NetworkUpdate::ChannelClosed { short_channel_id, is_permanent }) if $chan_closed => {
                                                assert_eq!(short_channel_id, $scid);
                                                assert!(is_permanent);
                                        },
-                                       _ => panic!("Unexpected update type"),
+                                       Some(_) => panic!("Unexpected update type"),
+                                       None => panic!("Expected update"),
                                }
                        },
                        _ => panic!("Unexpected event"),
@@ -1065,7 +1102,7 @@ macro_rules! expect_payment_failed {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentFailed { ref payment_hash, rejected_by_dest, ref error_code, ref error_data } => {
+                       Event::PaymentPathFailed { ref payment_hash, rejected_by_dest, network_update: _, ref error_code, ref error_data, .. } => {
                                assert_eq!(*payment_hash, $expected_payment_hash, "unexpected payment_hash");
                                assert_eq!(rejected_by_dest, $rejected_by_dest, "unexpected rejected_by_dest value");
                                assert!(error_code.is_some(), "expected error_code.is_some() = true");
@@ -1237,9 +1274,11 @@ pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, exp
 
                if !skip_last {
                        last_update_fulfill_dance!(origin_node, expected_route.first().unwrap());
-                       expect_payment_sent!(origin_node, our_payment_preimage);
                }
        }
+       if !skip_last {
+               expect_payment_sent!(origin_node, our_payment_preimage);
+       }
 }
 
 pub fn claim_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_preimage: PaymentPreimage) {
@@ -1250,7 +1289,7 @@ pub const TEST_FINAL_CLTV: u32 = 70;
 
 pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) {
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph,
                &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()),
                Some(&origin_node.node.list_usable_channels().iter().collect::<Vec<_>>()), &[],
                recv_value, TEST_FINAL_CLTV, origin_node.logger).unwrap();
@@ -1265,7 +1304,7 @@ pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route:
 
 pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64)  {
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, origin_node.logger).unwrap();
+       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph, &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, origin_node.logger).unwrap();
        assert_eq!(route.paths.len(), 1);
        assert_eq!(route.paths[0].len(), expected_route.len());
        for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
@@ -1282,77 +1321,100 @@ pub fn send_payment<'a, 'b, 'c>(origin: &Node<'a, 'b, 'c>, expected_route: &[&No
        claim_payment(&origin, expected_route, our_payment_preimage);
 }
 
-pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], skip_last: bool, our_payment_hash: PaymentHash)  {
-       assert!(expected_route.last().unwrap().node.fail_htlc_backwards(&our_payment_hash));
-       expect_pending_htlcs_forwardable!(expected_route.last().unwrap());
-       check_added_monitors!(expected_route.last().unwrap(), 1);
+pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash)  {
+       let mut expected_paths: Vec<_> = expected_paths_slice.iter().collect();
+       for path in expected_paths.iter() {
+               assert_eq!(path.last().unwrap().node.get_our_node_id(), expected_paths[0].last().unwrap().node.get_our_node_id());
+       }
+       assert!(expected_paths[0].last().unwrap().node.fail_htlc_backwards(&our_payment_hash));
+       expect_pending_htlcs_forwardable!(expected_paths[0].last().unwrap());
+       check_added_monitors!(expected_paths[0].last().unwrap(), expected_paths.len());
 
-       let mut next_msgs: Option<(msgs::UpdateFailHTLC, msgs::CommitmentSigned)> = None;
-       macro_rules! update_fail_dance {
-               ($node: expr, $prev_node: expr, $last_node: expr) => {
-                       {
-                               $node.node.handle_update_fail_htlc(&$prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
-                               commitment_signed_dance!($node, $prev_node, next_msgs.as_ref().unwrap().1, !$last_node);
-                               if skip_last && $last_node {
-                                       expect_pending_htlcs_forwardable!($node);
+       let mut per_path_msgs: Vec<((msgs::UpdateFailHTLC, msgs::CommitmentSigned), PublicKey)> = Vec::with_capacity(expected_paths.len());
+       let events = expected_paths[0].last().unwrap().node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), expected_paths.len());
+       for ev in events.iter() {
+               let (update_fail, commitment_signed, node_id) = match ev {
+                       &MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+                               assert!(update_add_htlcs.is_empty());
+                               assert!(update_fulfill_htlcs.is_empty());
+                               assert_eq!(update_fail_htlcs.len(), 1);
+                               assert!(update_fail_malformed_htlcs.is_empty());
+                               assert!(update_fee.is_none());
+                               (update_fail_htlcs[0].clone(), commitment_signed.clone(), node_id.clone())
+                       },
+                       _ => panic!("Unexpected event"),
+               };
+               per_path_msgs.push(((update_fail, commitment_signed), node_id));
+       }
+       per_path_msgs.sort_unstable_by(|(_, node_id_a), (_, node_id_b)| node_id_a.cmp(node_id_b));
+       expected_paths.sort_unstable_by(|path_a, path_b| path_a[path_a.len() - 2].node.get_our_node_id().cmp(&path_b[path_b.len() - 2].node.get_our_node_id()));
+
+       for (i, (expected_route, (path_msgs, next_hop))) in expected_paths.iter().zip(per_path_msgs.drain(..)).enumerate() {
+               let mut next_msgs = Some(path_msgs);
+               let mut expected_next_node = next_hop;
+               let mut prev_node = expected_route.last().unwrap();
+
+               for (idx, node) in expected_route.iter().rev().enumerate().skip(1) {
+                       assert_eq!(expected_next_node, node.node.get_our_node_id());
+                       let update_next_node = !skip_last || idx != expected_route.len() - 1;
+                       if next_msgs.is_some() {
+                               node.node.handle_update_fail_htlc(&prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
+                               commitment_signed_dance!(node, prev_node, next_msgs.as_ref().unwrap().1, update_next_node);
+                               if !update_next_node {
+                                       expect_pending_htlcs_forwardable!(node);
                                }
                        }
-               }
-       }
+                       let events = node.node.get_and_clear_pending_msg_events();
+                       if update_next_node {
+                               assert_eq!(events.len(), 1);
+                               match events[0] {
+                                       MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+                                               assert!(update_add_htlcs.is_empty());
+                                               assert!(update_fulfill_htlcs.is_empty());
+                                               assert_eq!(update_fail_htlcs.len(), 1);
+                                               assert!(update_fail_malformed_htlcs.is_empty());
+                                               assert!(update_fee.is_none());
+                                               expected_next_node = node_id.clone();
+                                               next_msgs = Some((update_fail_htlcs[0].clone(), commitment_signed.clone()));
+                                       },
+                                       _ => panic!("Unexpected event"),
+                               }
+                       } else {
+                               assert!(events.is_empty());
+                       }
+                       if !skip_last && idx == expected_route.len() - 1 {
+                               assert_eq!(expected_next_node, origin_node.node.get_our_node_id());
+                       }
 
-       let mut expected_next_node = expected_route.last().unwrap().node.get_our_node_id();
-       let mut prev_node = expected_route.last().unwrap();
-       for (idx, node) in expected_route.iter().rev().enumerate() {
-               assert_eq!(expected_next_node, node.node.get_our_node_id());
-               if next_msgs.is_some() {
-                       // We may be the "last node" for the purpose of the commitment dance if we're
-                       // skipping the last node (implying it is disconnected) and we're the
-                       // second-to-last node!
-                       update_fail_dance!(node, prev_node, skip_last && idx == expected_route.len() - 1);
+                       prev_node = node;
                }
 
-               let events = node.node.get_and_clear_pending_msg_events();
-               if !skip_last || idx != expected_route.len() - 1 {
+               if !skip_last {
+                       let prev_node = expected_route.first().unwrap();
+                       origin_node.node.handle_update_fail_htlc(&prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
+                       check_added_monitors!(origin_node, 0);
+                       assert!(origin_node.node.get_and_clear_pending_msg_events().is_empty());
+                       commitment_signed_dance!(origin_node, prev_node, next_msgs.as_ref().unwrap().1, false);
+                       let events = origin_node.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
                        match events[0] {
-                               MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
-                                       assert!(update_add_htlcs.is_empty());
-                                       assert!(update_fulfill_htlcs.is_empty());
-                                       assert_eq!(update_fail_htlcs.len(), 1);
-                                       assert!(update_fail_malformed_htlcs.is_empty());
-                                       assert!(update_fee.is_none());
-                                       expected_next_node = node_id.clone();
-                                       next_msgs = Some((update_fail_htlcs[0].clone(), commitment_signed.clone()));
+                               Event::PaymentPathFailed { payment_hash, rejected_by_dest, all_paths_failed, ref path, .. } => {
+                                       assert_eq!(payment_hash, our_payment_hash);
+                                       assert!(rejected_by_dest);
+                                       assert_eq!(all_paths_failed, i == expected_paths.len() - 1);
+                                       for (idx, hop) in expected_route.iter().enumerate() {
+                                               assert_eq!(hop.node.get_our_node_id(), path[idx].pubkey);
+                                       }
                                },
                                _ => panic!("Unexpected event"),
                        }
-               } else {
-                       assert!(events.is_empty());
-               }
-               if !skip_last && idx == expected_route.len() - 1 {
-                       assert_eq!(expected_next_node, origin_node.node.get_our_node_id());
-               }
-
-               prev_node = node;
-       }
-
-       if !skip_last {
-               update_fail_dance!(origin_node, expected_route.first().unwrap(), true);
-
-               let events = origin_node.node.get_and_clear_pending_events();
-               assert_eq!(events.len(), 1);
-               match events[0] {
-                       Event::PaymentFailed { payment_hash, rejected_by_dest, .. } => {
-                               assert_eq!(payment_hash, our_payment_hash);
-                               assert!(rejected_by_dest);
-                       },
-                       _ => panic!("Unexpected event"),
                }
        }
 }
 
-pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_hash: PaymentHash)  {
-       fail_payment_along_route(origin_node, expected_route, false, our_payment_hash);
+pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], our_payment_hash: PaymentHash)  {
+       fail_payment_along_route(origin_node, &[&expected_path[..]], false, our_payment_hash);
 }
 
 pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
@@ -1435,7 +1497,8 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
        let connect_style = Rc::new(RefCell::new(ConnectStyle::FullBlockViaListen));
 
        for i in 0..node_count {
-               let net_graph_msg_handler = NetGraphMsgHandler::new(cfgs[i].chain_source.genesis_hash, None, cfgs[i].logger);
+               let network_graph = NetworkGraph::new(cfgs[i].chain_source.genesis_hash);
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, cfgs[i].logger);
                nodes.push(Node{ chain_source: cfgs[i].chain_source,
                                 tx_broadcaster: cfgs[i].tx_broadcaster, chain_monitor: &cfgs[i].chain_monitor,
                                 keys_manager: &cfgs[i].keys_manager, node: &chan_mgrs[i], net_graph_msg_handler,
index 06cc80cc18b0a5b770be83ce25cbb82535d932ba..4b0016199de1a453cdd1caf88a7a8c56b5367443 100644 (file)
@@ -16,26 +16,25 @@ use chain::{Confirm, Listen, Watch};
 use chain::channelmonitor;
 use chain::channelmonitor::{ChannelMonitor, CLTV_CLAIM_BUFFER, LATENCY_GRACE_PERIOD_BLOCKS, ANTI_REORG_DELAY};
 use chain::transaction::OutPoint;
-use chain::keysinterface::{KeysInterface, BaseSign};
+use chain::keysinterface::BaseSign;
 use ln::{PaymentPreimage, PaymentSecret, PaymentHash};
 use ln::channel::{COMMITMENT_TX_BASE_WEIGHT, COMMITMENT_TX_WEIGHT_PER_HTLC};
-use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA};
+use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, MppId, RAACommitmentOrder, PaymentSendFailure, BREAKDOWN_TIMEOUT, MIN_CLTV_EXPIRY_DELTA};
 use ln::channel::{Channel, ChannelError};
 use ln::{chan_utils, onion_utils};
 use ln::chan_utils::HTLC_SUCCESS_TX_WEIGHT;
 use routing::router::{Route, RouteHop, RouteHint, RouteHintHop, get_route, get_keysend_route};
-use routing::network_graph::RoutingFees;
+use routing::network_graph::{NetworkUpdate, RoutingFees};
 use ln::features::{ChannelFeatures, InitFeatures, InvoiceFeatures, NodeFeatures};
 use ln::msgs;
-use ln::msgs::{ChannelMessageHandler,RoutingMessageHandler,HTLCFailChannelUpdate, ErrorAction};
+use ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
 use util::enforcing_trait_impls::EnforcingSigner;
 use util::{byte_utils, test_utils};
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose, ClosureReason};
 use util::errors::APIError;
 use util::ser::{Writeable, ReadableArgs};
 use util::config::UserConfig;
 
-use bitcoin::hashes::sha256d::Hash as Sha256dHash;
 use bitcoin::hash_types::{Txid, BlockHash};
 use bitcoin::blockdata::block::{Block, BlockHeader};
 use bitcoin::blockdata::script::Builder;
@@ -46,7 +45,7 @@ use bitcoin::network::constants::Network;
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hashes::Hash;
 
-use bitcoin::secp256k1::{Secp256k1, Message};
+use bitcoin::secp256k1::Secp256k1;
 use bitcoin::secp256k1::key::{PublicKey,SecretKey};
 
 use regex;
@@ -171,7 +170,7 @@ fn test_async_inbound_update_fee() {
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[0]);
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-       nodes[1].node.send_payment(&get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap(), our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap(), our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -272,7 +271,7 @@ fn test_update_fee_unordered_raa() {
        // ...but before it's delivered, nodes[1] starts to send a payment back to nodes[0]...
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[0]);
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-       nodes[1].node.send_payment(&get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap(), our_payment_hash, &Some(our_payment_secret)).unwrap();
+       nodes[1].node.send_payment(&get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap(), our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[1], 1);
 
        let payment_event = {
@@ -639,6 +638,7 @@ fn test_update_fee_that_funder_cannot_afford() {
        nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Funding remote cannot afford proposed new fee".to_string(), 1);
        check_added_monitors!(nodes[1], 1);
        check_closed_broadcast!(nodes[1], true);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: String::from("Funding remote cannot afford proposed new fee") });
 }
 
 #[test]
@@ -675,7 +675,7 @@ fn test_update_fee_with_fundee_update_add_htlc() {
 
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[0]);
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 800000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 800000, TEST_FINAL_CLTV, &logger).unwrap();
 
        // nothing happens since node[1] is in AwaitingRemoteRevoke
        nodes[1].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
@@ -739,6 +739,8 @@ fn test_update_fee_with_fundee_update_add_htlc() {
        send_payment(&nodes[1], &vec!(&nodes[0])[..], 800000);
        send_payment(&nodes[0], &vec!(&nodes[1])[..], 800000);
        close_channel(&nodes[0], &nodes[1], &chan.2, chan.3, true);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -851,6 +853,8 @@ fn test_update_fee() {
        assert_eq!(get_feerate!(nodes[0], channel_id), feerate + 30);
        assert_eq!(get_feerate!(nodes[1], channel_id), feerate + 30);
        close_channel(&nodes[0], &nodes[1], &chan.2, chan.3, true);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -978,10 +982,20 @@ fn fake_network_test() {
 
        // Close down the channels...
        close_channel(&nodes[0], &nodes[1], &chan_1.2, chan_1.3, true);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
        close_channel(&nodes[1], &nodes[2], &chan_2.2, chan_2.3, false);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[2], 1, ClosureReason::CooperativeClosure);
        close_channel(&nodes[2], &nodes[3], &chan_3.2, chan_3.3, true);
+       check_closed_event!(nodes[2], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[3], 1, ClosureReason::CooperativeClosure);
        close_channel(&nodes[1], &nodes[3], &chan_4.2, chan_4.3, false);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[3], 1, ClosureReason::CooperativeClosure);
        close_channel(&nodes[1], &nodes[3], &chan_5.2, chan_5.3, false);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[3], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -1001,7 +1015,7 @@ fn holding_cell_htlc_counting() {
        for _ in 0..::ln::channel::OUR_MAX_HTLCS {
                let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
                payments.push((payment_preimage, payment_hash));
        }
@@ -1018,7 +1032,7 @@ fn holding_cell_htlc_counting() {
        let (_, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[2]);
        {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), true, APIError::ChannelUnavailable { ref err },
                        assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
                assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
@@ -1029,7 +1043,7 @@ fn holding_cell_htlc_counting() {
        let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[2]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
                check_added_monitors!(nodes[0], 1);
        }
@@ -1051,8 +1065,7 @@ fn holding_cell_htlc_counting() {
        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_updates.update_fail_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], bs_fail_updates.commitment_signed, false, true);
 
-       expect_payment_failure_chan_update!(nodes[0], chan_2.0.contents.short_channel_id, false);
-       expect_payment_failed!(nodes[0], payment_hash_2, false);
+       expect_payment_failed_with_update!(nodes[0], payment_hash_2, false, chan_2.0.contents.short_channel_id, false);
 
        // Now forward all the pending HTLCs and claim them back
        nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &initial_payment_event.msgs[0]);
@@ -1154,7 +1167,7 @@ fn test_duplicate_htlc_different_direction_onchain() {
        let (payment_preimage, payment_hash, _) = route_payment(&nodes[0], &vec!(&nodes[1])[..], 900_000);
 
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 800_000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 800_000, TEST_FINAL_CLTV, &logger).unwrap();
        let node_a_payment_secret = nodes[0].node.create_inbound_payment_for_hash(payment_hash, None, 7200, 0).unwrap();
        send_along_route_with_secret(&nodes[1], route, &[&[&nodes[0]]], 800_000, payment_hash, node_a_payment_secret);
 
@@ -1178,6 +1191,7 @@ fn test_duplicate_htlc_different_direction_onchain() {
 
        mine_transaction(&nodes[0], &remote_txn[0]);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
 
        // Check we only broadcast 1 timeout tx
@@ -1240,7 +1254,7 @@ fn test_basic_channel_reserve() {
        let commit_tx_fee = 2 * commit_tx_fee_msat(get_feerate!(nodes[0], chan.2), 1 + 1);
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee;
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), max_can_send + 1, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), max_can_send + 1, TEST_FINAL_CLTV, &logger).unwrap();
        let err = nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).err().unwrap();
        match err {
                PaymentSendFailure::AllFailedRetrySafe(ref fails) => {
@@ -1459,6 +1473,7 @@ fn test_chan_reserve_violation_inbound_htlc_outbound_channel() {
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert_eq!(err_msg.data, "Cannot accept HTLC that would put our balance under counterparty-announced channel reserve value");
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "Cannot accept HTLC that would put our balance under counterparty-announced channel reserve value".to_string() });
 }
 
 #[test]
@@ -1482,7 +1497,7 @@ fn test_chan_reserve_dust_inbound_htlcs_outbound_chan() {
        push_amt -= Channel::<EnforcingSigner>::get_holder_selected_channel_reserve_satoshis(100_000) * 1000;
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, push_amt, InitFeatures::known(), InitFeatures::known());
 
-       let dust_amt = crate::ln::channel::MIN_DUST_LIMIT_SATOSHIS * 1000
+       let dust_amt = crate::ln::channel::MIN_CHAN_DUST_LIMIT_SATOSHIS * 1000
                + feerate_per_kw as u64 * HTLC_SUCCESS_TX_WEIGHT / 1000 * 1000 - 1;
        // In the previous code, routing this dust payment would cause nodes[0] to perceive a channel
        // reserve violation even though it's a dust HTLC and therefore shouldn't count towards the
@@ -1585,6 +1600,7 @@ fn test_chan_reserve_violation_inbound_htlc_inbound_chan() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert_eq!(err_msg.data, "Remote HTLC add would put them under remote reserve value");
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "Remote HTLC add would put them under remote reserve value".to_string() });
 }
 
 #[test]
@@ -1886,7 +1902,7 @@ fn channel_reserve_in_flight_removes() {
        let (payment_preimage_3, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[1]);
        let send_1 = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -1959,7 +1975,7 @@ fn channel_reserve_in_flight_removes() {
        let (payment_preimage_4, payment_hash_4, payment_secret_4) = get_payment_preimage_hash!(nodes[0]);
        let send_2 = {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 10000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 10000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4)).unwrap();
                check_added_monitors!(nodes[1], 1);
                let mut events = nodes[1].node.get_and_clear_pending_msg_events();
@@ -2041,6 +2057,8 @@ fn channel_monitor_network_test() {
        check_closed_broadcast!(nodes[0], true);
        assert_eq!(nodes[0].node.list_channels().len(), 0);
        assert_eq!(nodes[1].node.list_channels().len(), 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
+       check_closed_event!(nodes[1], 1, ClosureReason::DisconnectedPeer);
 
        // One pending HTLC is discarded by the force-close:
        let payment_preimage_1 = route_payment(&nodes[1], &vec!(&nodes[2], &nodes[3])[..], 3000000).0;
@@ -2061,6 +2079,8 @@ fn channel_monitor_network_test() {
        check_closed_broadcast!(nodes[2], true);
        assert_eq!(nodes[1].node.list_channels().len(), 0);
        assert_eq!(nodes[2].node.list_channels().len(), 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::DisconnectedPeer);
+       check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
 
        macro_rules! claim_funds {
                ($node: expr, $prev_node: expr, $preimage: expr) => {
@@ -2103,6 +2123,8 @@ fn channel_monitor_network_test() {
        check_closed_broadcast!(nodes[3], true);
        assert_eq!(nodes[2].node.list_channels().len(), 0);
        assert_eq!(nodes[3].node.list_channels().len(), 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::DisconnectedPeer);
+       check_closed_event!(nodes[3], 1, ClosureReason::CommitmentTxConfirmed);
 
        // Drop the ChannelMonitor for the previous channel to avoid it broadcasting transactions and
        // confusing us in the following tests.
@@ -2174,6 +2196,8 @@ fn channel_monitor_network_test() {
        assert_eq!(nodes[4].node.list_channels().len(), 0);
 
        nodes[3].chain_monitor.chain_monitor.monitors.write().unwrap().insert(OutPoint { txid: chan_3.3.txid(), index: 0 }, chan_3_mon);
+       check_closed_event!(nodes[3], 1, ClosureReason::CommitmentTxConfirmed);
+       check_closed_event!(nodes[4], 1, ClosureReason::CommitmentTxConfirmed);
 }
 
 #[test]
@@ -2223,6 +2247,7 @@ fn test_justice_tx() {
                        node_txn.truncate(1);
                }
                check_added_monitors!(nodes[1], 1);
+               check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                test_txn_broadcast(&nodes[1], &chan_5, None, HTLCType::NONE);
 
                mine_transaction(&nodes[0], &revoked_local_txn[0]);
@@ -2230,6 +2255,7 @@ fn test_justice_tx() {
                // Verify broadcast of revoked HTLC-timeout
                let node_txn = test_txn_broadcast(&nodes[0], &chan_5, Some(revoked_local_txn[0].clone()), HTLCType::TIMEOUT);
                check_added_monitors!(nodes[0], 1);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                // Broadcast revoked HTLC-timeout on node 1
                mine_transaction(&nodes[1], &node_txn[1]);
                test_revoked_htlc_claim_txn_broadcast(&nodes[1], node_txn[1].clone(), revoked_local_txn[0].clone());
@@ -2271,9 +2297,11 @@ fn test_justice_tx() {
                test_txn_broadcast(&nodes[0], &chan_6, None, HTLCType::NONE);
 
                mine_transaction(&nodes[1], &revoked_local_txn[0]);
+               check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                let node_txn = test_txn_broadcast(&nodes[1], &chan_6, Some(revoked_local_txn[0].clone()), HTLCType::SUCCESS);
                check_added_monitors!(nodes[1], 1);
                mine_transaction(&nodes[0], &node_txn[1]);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                test_revoked_htlc_claim_txn_broadcast(&nodes[0], node_txn[1].clone(), revoked_local_txn[0].clone());
        }
        get_announce_close_broadcast_events(&nodes, 0, 1);
@@ -2301,6 +2329,7 @@ fn revoked_output_claim() {
        // Inform nodes[1] that nodes[0] broadcast a stale tx
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 2); // ChannelMonitor: justice tx against revoked to_local output, ChannelManager: local commitment tx
 
@@ -2310,7 +2339,8 @@ fn revoked_output_claim() {
        // Inform nodes[0] that a watchtower cheated on its behalf, so it will force-close the chan
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
        get_announce_close_broadcast_events(&nodes, 0, 1);
-       check_added_monitors!(nodes[0], 1)
+       check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
 }
 
 #[test]
@@ -2347,8 +2377,10 @@ fn claim_htlc_outputs_shared_tx() {
        {
                mine_transaction(&nodes[0], &revoked_local_txn[0]);
                check_added_monitors!(nodes[0], 1);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                mine_transaction(&nodes[1], &revoked_local_txn[0]);
                check_added_monitors!(nodes[1], 1);
+               check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                expect_payment_failed!(nodes[1], payment_hash_2, true);
 
@@ -2405,7 +2437,13 @@ fn claim_htlc_outputs_single_tx() {
                check_added_monitors!(nodes[0], 1);
                confirm_transaction_at(&nodes[1], &revoked_local_txn[0], 100);
                check_added_monitors!(nodes[1], 1);
-               expect_pending_htlcs_forwardable_ignore!(nodes[0]);
+               check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
+               let mut events = nodes[0].node.get_and_clear_pending_events();
+               expect_pending_htlcs_forwardable_from_events!(nodes[0], events[0..1], true);
+               match events[1] {
+                       Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}
+                       _ => panic!("Unexpected event"),
+               }
 
                connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
                expect_payment_failed!(nodes[1], payment_hash_2, true);
@@ -2503,6 +2541,7 @@ fn test_htlc_on_chain_success() {
        mine_transaction(&nodes[2], &commitment_tx[0]);
        check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 3 (commitment tx, 2*htlc-success tx), ChannelMonitor : 2 (2 * HTLC-Success tx)
        assert_eq!(node_txn.len(), 5);
        assert_eq!(node_txn[0], node_txn[3]);
@@ -2528,11 +2567,15 @@ fn test_htlc_on_chain_success() {
                added_monitors.clear();
        }
        let forwarded_events = nodes[1].node.get_and_clear_pending_events();
-       assert_eq!(forwarded_events.len(), 2);
-       if let Event::PaymentForwarded { fee_earned_msat: Some(1000), claim_from_onchain_tx: true } = forwarded_events[0] {
-               } else { panic!(); }
+       assert_eq!(forwarded_events.len(), 3);
+       match forwarded_events[0] {
+               Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}
+               _ => panic!("Unexpected event"),
+       }
        if let Event::PaymentForwarded { fee_earned_msat: Some(1000), claim_from_onchain_tx: true } = forwarded_events[1] {
                } else { panic!(); }
+       if let Event::PaymentForwarded { fee_earned_msat: Some(1000), claim_from_onchain_tx: true } = forwarded_events[2] {
+               } else { panic!(); }
        let events = nodes[1].node.get_and_clear_pending_msg_events();
        {
                let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
@@ -2599,6 +2642,7 @@ fn test_htlc_on_chain_success() {
        mine_transaction(&nodes[1], &node_a_commitment_tx[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
        assert_eq!(node_txn.len(), 6); // ChannelManager : 3 (commitment tx + HTLC-Sucess * 2), ChannelMonitor : 3 (HTLC-Success, 2* RBF bumps of above HTLC txn)
        let commitment_spend =
@@ -2634,7 +2678,7 @@ fn test_htlc_on_chain_success() {
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
        let events = nodes[0].node.get_and_clear_pending_events();
-       assert_eq!(events.len(), 2);
+       assert_eq!(events.len(), 3);
        let mut first_claimed = false;
        for event in events {
                match event {
@@ -2646,6 +2690,7 @@ fn test_htlc_on_chain_success() {
                                        assert_eq!(payment_preimage, our_payment_preimage_2);
                                }
                        },
+                       Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {},
                        _ => panic!("Unexpected event"),
                }
        }
@@ -2702,6 +2747,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
        mine_transaction(&nodes[2], &commitment_tx[0]);
        check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 1 (commitment tx)
        assert_eq!(node_txn.len(), 1);
        check_spends!(node_txn[0], chan_2.3);
@@ -2711,6 +2757,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
        // Verify that B's ChannelManager is able to detect that HTLC is timeout by its own tx and react backward in consequence
        connect_blocks(&nodes[1], 200 - nodes[2].best_block_info().1);
        mine_transaction(&nodes[1], &commitment_tx[0]);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let timeout_tx;
        {
                let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -2778,6 +2825,7 @@ fn do_test_htlc_on_chain_timeout(connect_style: ConnectStyle) {
 
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 1 commitment tx, ChannelMonitor : 1 timeout tx
        assert_eq!(node_txn.len(), 2);
        check_spends!(node_txn[0], chan_1.3);
@@ -2816,6 +2864,7 @@ fn test_simple_commitment_revoked_fail_backward() {
        let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 3000000);
 
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
        check_added_monitors!(nodes[1], 1);
        check_closed_broadcast!(nodes[1], true);
@@ -2834,8 +2883,7 @@ fn test_simple_commitment_revoked_fail_backward() {
 
                        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
                        commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false, true);
-                       expect_payment_failure_chan_update!(nodes[0], chan_2.0.contents.short_channel_id, true);
-                       expect_payment_failed!(nodes[0], payment_hash, false);
+                       expect_payment_failed_with_update!(nodes[0], payment_hash, false, chan_2.0.contents.short_channel_id, true);
                },
                _ => panic!("Unexpected event"),
        }
@@ -2937,7 +2985,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        let (_, fourth_payment_hash, fourth_payment_secret) = get_payment_preimage_hash!(nodes[2]);
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[1].node.send_payment(&route, fourth_payment_hash, &Some(fourth_payment_secret)).unwrap();
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
        assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
@@ -2966,15 +3014,19 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 
        let events = nodes[1].node.get_and_clear_pending_events();
-       assert_eq!(events.len(), if deliver_bs_raa { 1 } else { 2 });
+       assert_eq!(events.len(), if deliver_bs_raa { 2 } else { 3 });
        match events[0] {
-               Event::PaymentFailed { ref payment_hash, .. } => {
+               Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => { },
+               _ => panic!("Unexepected event"),
+       }
+       match events[1] {
+               Event::PaymentPathFailed { ref payment_hash, .. } => {
                        assert_eq!(*payment_hash, fourth_payment_hash);
                },
                _ => panic!("Unexpected event"),
        }
        if !deliver_bs_raa {
-               match events[1] {
+               match events[2] {
                        Event::PendingHTLCsForwardable { .. } => { },
                        _ => panic!("Unexpected event"),
                };
@@ -3021,33 +3073,30 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use
 
                        commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false, true);
 
-                       let events = nodes[0].node.get_and_clear_pending_msg_events();
-                       // If we delivered B's RAA we got an unknown preimage error, not something
-                       // that we should update our routing table for.
-                       assert_eq!(events.len(), if deliver_bs_raa { 2 } else { 3 });
-                       for event in events {
-                               match event {
-                                       MessageSendEvent::PaymentFailureNetworkUpdate { .. } => {},
-                                       _ => panic!("Unexpected event"),
-                               }
-                       }
                        let events = nodes[0].node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 3);
                        match events[0] {
-                               Event::PaymentFailed { ref payment_hash, .. } => {
+                               Event::PaymentPathFailed { ref payment_hash, rejected_by_dest: _, ref network_update, .. } => {
                                        assert!(failed_htlcs.insert(payment_hash.0));
+                                       // If we delivered B's RAA we got an unknown preimage error, not something
+                                       // that we should update our routing table for.
+                                       if !deliver_bs_raa {
+                                               assert!(network_update.is_some());
+                                       }
                                },
                                _ => panic!("Unexpected event"),
                        }
                        match events[1] {
-                               Event::PaymentFailed { ref payment_hash, .. } => {
+                               Event::PaymentPathFailed { ref payment_hash, rejected_by_dest: _, ref network_update, .. } => {
                                        assert!(failed_htlcs.insert(payment_hash.0));
+                                       assert!(network_update.is_some());
                                },
                                _ => panic!("Unexpected event"),
                        }
                        match events[2] {
-                               Event::PaymentFailed { ref payment_hash, .. } => {
+                               Event::PaymentPathFailed { ref payment_hash, rejected_by_dest: _, ref network_update, .. } => {
                                        assert!(failed_htlcs.insert(payment_hash.0));
+                                       assert!(network_update.is_some());
                                },
                                _ => panic!("Unexpected event"),
                        }
@@ -3089,7 +3138,7 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        {
                let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[1]);
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
@@ -3106,7 +3155,7 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
        let (_, failed_payment_hash, failed_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, failed_payment_hash, &Some(failed_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 0);
 
@@ -3121,7 +3170,7 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
                let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
                let current_height = nodes[1].node.best_block.read().unwrap().height() + 1;
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, TEST_FINAL_CLTV, &logger).unwrap();
                let (onion_payloads, _amount_msat, cltv_expiry) = onion_utils::build_onion_payloads(&route.paths[0], 50_000, &Some(payment_secret), current_height, &None).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&secp_ctx, &route.paths[0], &session_priv).unwrap();
                let onion_routing_packet = onion_utils::construct_onion_packet(onion_payloads, onion_keys, [0; 32], &payment_hash);
@@ -3137,9 +3186,21 @@ fn fail_backward_pending_htlc_upon_channel_failure() {
                };
                nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &update_add_htlc);
        }
-
+       let events = nodes[0].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 2);
        // Check that Alice fails backward the pending HTLC from the second payment.
-       expect_payment_failed!(nodes[0], failed_payment_hash, true);
+       match events[0] {
+               Event::PaymentPathFailed { payment_hash, .. } => {
+                       assert_eq!(payment_hash, failed_payment_hash);
+               },
+               _ => panic!("Unexpected event"),
+       }
+       match events[1] {
+               Event::ChannelClosed { reason: ClosureReason::ProcessingError { ref err }, .. } => {
+                       assert_eq!(err, "Remote side tried to send a 0-msat HTLC");
+               },
+               _ => panic!("Unexpected event {:?}", events[1]),
+       }
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
 }
@@ -3159,6 +3220,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
        connect_blocks(&nodes[0], TEST_FINAL_CLTV + LATENCY_GRACE_PERIOD_BLOCKS + 1);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 3);
@@ -3168,6 +3230,7 @@ fn test_htlc_ignore_latest_remote_commitment() {
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[0].clone(), node_txn[1].clone()]});
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        // Duplicate the connect_block call since this may happen due to other listeners
        // registering new transactions
@@ -3190,7 +3253,7 @@ fn test_force_close_fail_back() {
 
        let mut payment_event = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, 42, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, 42, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
@@ -3222,6 +3285,7 @@ fn test_force_close_fail_back() {
        nodes[2].node.force_close_channel(&payment_event.commitment_msg.channel_id).unwrap();
        check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
        let tx = {
                let mut node_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap();
                // Note that we don't bother broadcasting the HTLC-Success transaction here as we don't
@@ -3236,6 +3300,7 @@ fn test_force_close_fail_back() {
        // Note no UpdateHTLCs event here from nodes[1] to nodes[0]!
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        // Now check that if we add the preimage to ChannelMonitor it broadcasts our HTLC-Success..
        {
@@ -3314,7 +3379,7 @@ fn test_simple_peer_disconnect() {
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 
        claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], true, payment_preimage_3);
-       fail_payment_along_route(&nodes[0], &[&nodes[1], &nodes[2]], true, payment_hash_5);
+       fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[2]]], true, payment_hash_5);
 
        reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (1, 0), (1, 0), (false, false));
        {
@@ -3327,7 +3392,7 @@ fn test_simple_peer_disconnect() {
                        _ => panic!("Unexpected event"),
                }
                match events[1] {
-                       Event::PaymentFailed { payment_hash, rejected_by_dest, .. } => {
+                       Event::PaymentPathFailed { payment_hash, rejected_by_dest, .. } => {
                                assert_eq!(payment_hash, payment_hash_5);
                                assert!(rejected_by_dest);
                        },
@@ -3362,7 +3427,7 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
        let logger = test_utils::TestLogger::new();
        let payment_event = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(),
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph,
                        &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
                        &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
@@ -3557,7 +3622,7 @@ fn do_test_drop_messages_peer_disconnect(messages_delivered: u8, simulate_broken
 
        // Channel should still work fine...
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph,
                &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), Some(&nodes[0].node.list_usable_channels().iter().collect::<Vec<_>>()),
                &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        let payment_preimage_2 = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
@@ -3664,7 +3729,7 @@ fn test_funding_peer_disconnect() {
 
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        let (payment_preimage, _, _) = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000);
        claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 
@@ -3740,7 +3805,7 @@ fn test_drop_messages_peer_disconnect_dual_htlc() {
        // Now try to send a second payment which will fail to send
        let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
@@ -3887,12 +3952,13 @@ fn do_test_htlc_timeout(send_partial_mpp: bool) {
 
        let our_payment_hash = if send_partial_mpp {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                let (_, our_payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[1]);
                // Use the utility function send_payment_along_path to send the payment with MPP data which
                // indicates there are more HTLCs coming.
                let cur_height = CHAN_CONFIRM_DEPTH + 1; // route_payment calls send_payment, which adds 1 to the current height. So we do the same here to match.
-               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200000, cur_height, &None).unwrap();
+               let mpp_id = MppId([42; 32]);
+               nodes[0].node.send_payment_along_path(&route.paths[0], &our_payment_hash, &Some(payment_secret), 200000, cur_height, mpp_id, &None).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
                assert_eq!(events.len(), 1);
@@ -3960,7 +4026,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
        let (_, first_payment_hash, first_payment_secret) = get_payment_preimage_hash!(nodes[2]);
        {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, first_payment_hash, &Some(first_payment_secret)).unwrap();
        }
        assert_eq!(nodes[1].node.get_and_clear_pending_msg_events().len(), 1);
@@ -3970,7 +4036,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
        let (_, second_payment_hash, second_payment_secret) = get_payment_preimage_hash!(nodes[2]);
        if forwarded_htlc {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, second_payment_hash, &Some(first_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let payment_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
@@ -3980,7 +4046,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
                check_added_monitors!(nodes[1], 0);
        } else {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, second_payment_hash, &Some(second_payment_secret)).unwrap();
                check_added_monitors!(nodes[1], 0);
        }
@@ -4002,8 +4068,7 @@ fn do_test_holding_cell_htlc_add_timeouts(forwarded_htlc: bool) {
                        },
                        _ => unreachable!(),
                }
-               expect_payment_failed!(nodes[0], second_payment_hash, false);
-               expect_payment_failure_chan_update!(nodes[0], chan_2.0.contents.short_channel_id, false);
+               expect_payment_failed_with_update!(nodes[0], second_payment_hash, false, chan_2.0.contents.short_channel_id, false);
        } else {
                expect_payment_failed!(nodes[1], second_payment_hash, true);
        }
@@ -4015,83 +4080,6 @@ fn test_holding_cell_htlc_add_timeouts() {
        do_test_holding_cell_htlc_add_timeouts(true);
 }
 
-#[test]
-fn test_invalid_channel_announcement() {
-       //Test BOLT 7 channel_announcement msg requirement for final node, gather data to build customed channel_announcement msgs
-       let secp_ctx = Secp256k1::new();
-       let chanmon_cfgs = create_chanmon_cfgs(2);
-       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
-       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
-
-       let chan_announcement = create_chan_between_nodes(&nodes[0], &nodes[1], InitFeatures::known(), InitFeatures::known());
-
-       let a_channel_lock = nodes[0].node.channel_state.lock().unwrap();
-       let b_channel_lock = nodes[1].node.channel_state.lock().unwrap();
-       let as_chan = a_channel_lock.by_id.get(&chan_announcement.3).unwrap();
-       let bs_chan = b_channel_lock.by_id.get(&chan_announcement.3).unwrap();
-
-       nodes[0].net_graph_msg_handler.handle_htlc_fail_channel_update(&msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id : as_chan.get_short_channel_id().unwrap(), is_permanent: false } );
-
-       let as_bitcoin_key = as_chan.get_signer().inner.holder_channel_pubkeys.funding_pubkey;
-       let bs_bitcoin_key = bs_chan.get_signer().inner.holder_channel_pubkeys.funding_pubkey;
-
-       let as_network_key = nodes[0].node.get_our_node_id();
-       let bs_network_key = nodes[1].node.get_our_node_id();
-
-       let were_node_one = as_bitcoin_key.serialize()[..] < bs_bitcoin_key.serialize()[..];
-
-       let mut chan_announcement;
-
-       macro_rules! dummy_unsigned_msg {
-               () => {
-                       msgs::UnsignedChannelAnnouncement {
-                               features: ChannelFeatures::known(),
-                               chain_hash: genesis_block(Network::Testnet).header.block_hash(),
-                               short_channel_id: as_chan.get_short_channel_id().unwrap(),
-                               node_id_1: if were_node_one { as_network_key } else { bs_network_key },
-                               node_id_2: if were_node_one { bs_network_key } else { as_network_key },
-                               bitcoin_key_1: if were_node_one { as_bitcoin_key } else { bs_bitcoin_key },
-                               bitcoin_key_2: if were_node_one { bs_bitcoin_key } else { as_bitcoin_key },
-                               excess_data: Vec::new(),
-                       }
-               }
-       }
-
-       macro_rules! sign_msg {
-               ($unsigned_msg: expr) => {
-                       let msghash = Message::from_slice(&Sha256dHash::hash(&$unsigned_msg.encode()[..])[..]).unwrap();
-                       let as_bitcoin_sig = secp_ctx.sign(&msghash, &as_chan.get_signer().inner.funding_key);
-                       let bs_bitcoin_sig = secp_ctx.sign(&msghash, &bs_chan.get_signer().inner.funding_key);
-                       let as_node_sig = secp_ctx.sign(&msghash, &nodes[0].keys_manager.get_node_secret());
-                       let bs_node_sig = secp_ctx.sign(&msghash, &nodes[1].keys_manager.get_node_secret());
-                       chan_announcement = msgs::ChannelAnnouncement {
-                               node_signature_1 : if were_node_one { as_node_sig } else { bs_node_sig},
-                               node_signature_2 : if were_node_one { bs_node_sig } else { as_node_sig},
-                               bitcoin_signature_1: if were_node_one { as_bitcoin_sig } else { bs_bitcoin_sig },
-                               bitcoin_signature_2 : if were_node_one { bs_bitcoin_sig } else { as_bitcoin_sig },
-                               contents: $unsigned_msg
-                       }
-               }
-       }
-
-       let unsigned_msg = dummy_unsigned_msg!();
-       sign_msg!(unsigned_msg);
-       assert_eq!(nodes[0].net_graph_msg_handler.handle_channel_announcement(&chan_announcement).unwrap(), true);
-       let _ = nodes[0].net_graph_msg_handler.handle_htlc_fail_channel_update(&msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id : as_chan.get_short_channel_id().unwrap(), is_permanent: false } );
-
-       // Configured with Network::Testnet
-       let mut unsigned_msg = dummy_unsigned_msg!();
-       unsigned_msg.chain_hash = genesis_block(Network::Bitcoin).header.block_hash();
-       sign_msg!(unsigned_msg);
-       assert!(nodes[0].net_graph_msg_handler.handle_channel_announcement(&chan_announcement).is_err());
-
-       let mut unsigned_msg = dummy_unsigned_msg!();
-       unsigned_msg.chain_hash = BlockHash::hash(&[1,2,3,4,5,6,7,8,9]);
-       sign_msg!(unsigned_msg);
-       assert!(nodes[0].net_graph_msg_handler.handle_channel_announcement(&chan_announcement).is_err());
-}
-
 #[test]
 fn test_no_txn_manager_serialize_deserialize() {
        let chanmon_cfgs = create_chanmon_cfgs(2);
@@ -4167,6 +4155,34 @@ fn test_no_txn_manager_serialize_deserialize() {
        send_payment(&nodes[0], &[&nodes[1]], 1000000);
 }
 
+#[test]
+fn mpp_failure() {
+       let chanmon_cfgs = create_chanmon_cfgs(4);
+       let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
+       let nodes = create_network(4, &node_cfgs, &node_chanmgrs);
+
+       let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
+       let chan_2_id = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
+       let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
+       let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
+       let logger = test_utils::TestLogger::new();
+
+       let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[3]);
+       let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
+       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let path = route.paths[0].clone();
+       route.paths.push(path);
+       route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
+       route.paths[0][0].short_channel_id = chan_1_id;
+       route.paths[0][1].short_channel_id = chan_3_id;
+       route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
+       route.paths[1][0].short_channel_id = chan_2_id;
+       route.paths[1][1].short_channel_id = chan_4_id;
+       send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], 200_000, payment_hash, payment_secret);
+       fail_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_hash);
+}
+
 #[test]
 fn test_dup_htlc_onchain_fails_on_reload() {
        // When a Channel is closed, any outbound HTLCs which were relayed through it are simply
@@ -4176,7 +4192,7 @@ fn test_dup_htlc_onchain_fails_on_reload() {
        //
        // If, due to an on-chain event, an HTLC is failed/claimed, and then we serialize the
        // ChannelManager, we generally expect there not to be a duplicate HTLC fail/claim (eg via a
-       // PaymentFailed event appearing). However, because we may not serialize the relevant
+       // PaymentPathFailed event appearing). However, because we may not serialize the relevant
        // ChannelMonitor at the same time, this isn't strictly guaranteed. In order to provide this
        // consistency, the ChannelManager explicitly tracks pending-onchain-resolution outbound HTLCs
        // and de-duplicates ChannelMonitor events.
@@ -4198,6 +4214,7 @@ fn test_dup_htlc_onchain_fails_on_reload() {
        nodes[0].node.force_close_channel(&nodes[0].node.list_channels()[0].channel_id).unwrap();
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 
        nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
        nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
@@ -4215,6 +4232,7 @@ fn test_dup_htlc_onchain_fails_on_reload() {
        connect_block(&nodes[1], &Block { header, txdata: vec![node_txn[1].clone(), node_txn[2].clone()]});
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let claim_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 
        header.prev_blockhash = nodes[0].best_block_hash();
@@ -4559,6 +4577,7 @@ fn test_manager_serialize_deserialize_inconsistent_monitor() {
                check_added_monitors!(nodes[0], 1);
        }
        nodes[0].node = &nodes_0_deserialized;
+       check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 
        // nodes[1] and nodes[2] have no lost state with nodes[0]...
        reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
@@ -4622,6 +4641,7 @@ fn test_claim_sizeable_push_msat() {
        nodes[1].node.force_close_channel(&chan.2).unwrap();
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 1);
        check_spends!(node_txn[0], chan.3);
@@ -4650,6 +4670,7 @@ fn test_claim_on_remote_sizeable_push_msat() {
        nodes[0].node.force_close_channel(&chan.2).unwrap();
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 1);
@@ -4659,6 +4680,7 @@ fn test_claim_on_remote_sizeable_push_msat() {
        mine_transaction(&nodes[1], &node_txn[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 
        let spend_txn = check_spendable_outputs!(nodes[1], node_cfgs[1].keys_manager);
@@ -4686,6 +4708,7 @@ fn test_claim_on_remote_revoked_sizeable_push_msat() {
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        mine_transaction(&nodes[1], &node_txn[0]);
@@ -4738,6 +4761,7 @@ fn test_static_spendable_outputs_preimage_tx() {
        check_spends!(node_txn[2], node_txn[1]);
 
        mine_transaction(&nodes[1], &node_txn[0]);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 
        let spend_txn = check_spendable_outputs!(nodes[1], node_cfgs[1].keys_manager);
@@ -4782,6 +4806,7 @@ fn test_static_spendable_outputs_timeout_tx() {
        assert_eq!(node_txn[1].input[0].witness.last().unwrap().len(), ACCEPTED_HTLC_SCRIPT_WEIGHT);
 
        mine_transaction(&nodes[1], &node_txn[1]);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
        expect_payment_failed!(nodes[1], our_payment_hash, true);
 
@@ -4812,6 +4837,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_commitment_tx() {
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 2);
@@ -4848,6 +4874,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() {
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
 
        let revoked_htlc_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -4863,6 +4890,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_timeout_tx() {
        connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[1].clone()] });
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 3); // ChannelMonitor: bogus justice tx, justice tx on revoked outputs, ChannelManager: local commitment tx
@@ -4919,6 +4947,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_success_tx() {
        mine_transaction(&nodes[1], &revoked_local_txn[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
 
        assert_eq!(revoked_htlc_txn.len(), 2);
@@ -4935,6 +4964,7 @@ fn test_static_spendable_outputs_justice_tx_revoked_htlc_success_tx() {
        connect_block(&nodes[0], &Block { header, txdata: vec![revoked_local_txn[0].clone(), revoked_htlc_txn[0].clone()] });
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
 
        let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
        assert_eq!(node_txn.len(), 3); // ChannelMonitor: justice tx on revoked commitment, justice tx on revoked HTLC-success, ChannelManager: local commitment tx
@@ -5015,6 +5045,7 @@ fn test_onchain_to_onchain_claim() {
        mine_transaction(&nodes[2], &commitment_tx[0]);
        check_closed_broadcast!(nodes[2], true);
        check_added_monitors!(nodes[2], 1);
+       check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
 
        let c_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().clone(); // ChannelManager : 2 (commitment tx, HTLC-Success tx), ChannelMonitor : 1 (HTLC-Success tx)
        assert_eq!(c_txn.len(), 3);
@@ -5031,7 +5062,19 @@ fn test_onchain_to_onchain_claim() {
        let header = BlockHeader { version: 0x20000000, prev_blockhash: nodes[1].best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42};
        connect_block(&nodes[1], &Block { header, txdata: vec![c_txn[1].clone(), c_txn[2].clone()]});
        check_added_monitors!(nodes[1], 1);
-       expect_payment_forwarded!(nodes[1], Some(1000), true);
+       let events = nodes[1].node.get_and_clear_pending_events();
+       assert_eq!(events.len(), 2);
+       match events[0] {
+               Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}
+               _ => panic!("Unexpected event"),
+       }
+       match events[1] {
+               Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => {
+                       assert_eq!(fee_earned_msat, Some(1000));
+                       assert_eq!(claim_from_onchain_tx, true);
+               },
+               _ => panic!("Unexpected event"),
+       }
        {
                let mut b_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                // ChannelMonitor: claim tx
@@ -5039,9 +5082,9 @@ fn test_onchain_to_onchain_claim() {
                check_spends!(b_txn[0], chan_2.3); // B local commitment tx, issued by ChannelManager
                b_txn.clear();
        }
+       check_added_monitors!(nodes[1], 1);
        let msg_events = nodes[1].node.get_and_clear_pending_msg_events();
        assert_eq!(msg_events.len(), 3);
-       check_added_monitors!(nodes[1], 1);
        match msg_events[0] {
                MessageSendEvent::BroadcastChannelUpdate { .. } => {},
                _ => panic!("Unexpected event"),
@@ -5063,6 +5106,7 @@ fn test_onchain_to_onchain_claim() {
        // Broadcast A's commitment tx on B's chain to see if we are able to claim inbound HTLC with our HTLC-Success tx
        let commitment_tx = get_local_commitment_txn!(nodes[0], chan_1.2);
        mine_transaction(&nodes[1], &commitment_tx[0]);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let b_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
        // ChannelMonitor: HTLC-Success tx, ChannelManager: local commitment tx + HTLC-Success tx
        assert_eq!(b_txn.len(), 3);
@@ -5109,7 +5153,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        // We reduce the final CLTV here by a somewhat arbitrary constant to keep it under the one-byte
        // script push size limit so that the below script length checks match
        // ACCEPTED_HTLC_SCRIPT_WEIGHT.
-       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph,
                &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 900000, TEST_FINAL_CLTV - 40, nodes[0].logger).unwrap();
        send_along_route_with_secret(&nodes[0], route, &[&[&nodes[1], &nodes[2], &nodes[3]]], 900000, duplicate_payment_hash, payment_secret);
 
@@ -5120,6 +5164,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        mine_transaction(&nodes[1], &commitment_txn[0]);
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], TEST_FINAL_CLTV - 40 + MIN_CLTV_EXPIRY_DELTA as u32 - 1); // Confirm blocks until the HTLC expires
 
        let htlc_timeout_tx;
@@ -5146,6 +5191,7 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        nodes[2].node.claim_funds(our_payment_preimage);
        mine_transaction(&nodes[2], &commitment_txn[0]);
        check_added_monitors!(nodes[2], 2);
+       check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
        let events = nodes[2].node.get_and_clear_pending_msg_events();
        match events[0] {
                MessageSendEvent::UpdateHTLCs { .. } => {},
@@ -5184,9 +5230,8 @@ fn test_duplicate_payment_hash_one_failure_one_success() {
        assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
        {
                commitment_signed_dance!(nodes[0], nodes[1], &htlc_updates.commitment_signed, false, true);
-               expect_payment_failure_chan_update!(nodes[0], chan_2.0.contents.short_channel_id, true);
        }
-       expect_payment_failed!(nodes[0], duplicate_payment_hash, false);
+       expect_payment_failed_with_update!(nodes[0], duplicate_payment_hash, false, chan_2.0.contents.short_channel_id, true);
 
        // Solve 2nd HTLC by broadcasting on B's chain HTLC-Success Tx from C
        // Note that the fee paid is effectively double as the HTLC value (including the nodes[1] fee
@@ -5234,6 +5279,7 @@ fn test_dynamic_spendable_outputs_local_htlc_success_tx() {
        check_added_monitors!(nodes[1], 1);
        mine_transaction(&nodes[1], &local_txn[0]);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        let events = nodes[1].node.get_and_clear_pending_msg_events();
        match events[0] {
                MessageSendEvent::UpdateHTLCs { .. } => {},
@@ -5307,7 +5353,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        let (_, payment_hash_2, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], ds_dust_limit*1000); // not added < dust limit + HTLC tx fee
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
        let our_node_id = &nodes[1].node.get_our_node_id();
-       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), ds_dust_limit*1000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph, &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), ds_dust_limit*1000, TEST_FINAL_CLTV, &logger).unwrap();
        // 2nd HTLC:
        send_along_route_with_secret(&nodes[1], route.clone(), &[&[&nodes[2], &nodes[3], &nodes[5]]], ds_dust_limit*1000, payment_hash_1, nodes[5].node.create_inbound_payment_for_hash(payment_hash_1, None, 7200, 0).unwrap()); // not added < dust limit + HTLC tx fee
        // 3rd HTLC:
@@ -5316,7 +5362,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        let (_, payment_hash_3, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], 1000000);
        // 5th HTLC:
        let (_, payment_hash_4, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], 1000000);
-       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph, &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        // 6th HTLC:
        send_along_route_with_secret(&nodes[1], route.clone(), &[&[&nodes[2], &nodes[3], &nodes[5]]], 1000000, payment_hash_3, nodes[5].node.create_inbound_payment_for_hash(payment_hash_3, None, 7200, 0).unwrap());
        // 7th HTLC:
@@ -5325,13 +5371,13 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        // 8th HTLC:
        let (_, payment_hash_5, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], 1000000);
        // 9th HTLC:
-       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), ds_dust_limit*1000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph, &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), ds_dust_limit*1000, TEST_FINAL_CLTV, &logger).unwrap();
        send_along_route_with_secret(&nodes[1], route, &[&[&nodes[2], &nodes[3], &nodes[5]]], ds_dust_limit*1000, payment_hash_5, nodes[5].node.create_inbound_payment_for_hash(payment_hash_5, None, 7200, 0).unwrap()); // not added < dust limit + HTLC tx fee
 
        // 10th HTLC:
        let (_, payment_hash_6, _) = route_payment(&nodes[0], &[&nodes[2], &nodes[3], &nodes[4]], ds_dust_limit*1000); // not added < dust limit + HTLC tx fee
        // 11th HTLC:
-       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(our_node_id, &net_graph_msg_handler.network_graph, &nodes[5].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        send_along_route_with_secret(&nodes[1], route, &[&[&nodes[2], &nodes[3], &nodes[5]]], 1000000, payment_hash_6, nodes[5].node.create_inbound_payment_for_hash(payment_hash_6, None, 7200, 0).unwrap());
 
        // Double-check that six of the new HTLC were added
@@ -5404,9 +5450,26 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        } else {
                mine_transaction(&nodes[2], &ds_prev_commitment_tx[0]);
        }
+       let events = nodes[2].node.get_and_clear_pending_events();
+       let close_event = if deliver_last_raa {
+               assert_eq!(events.len(), 2);
+               events[1].clone()
+       } else {
+               assert_eq!(events.len(), 1);
+               events[0].clone()
+       };
+       match close_event {
+               Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}
+               _ => panic!("Unexpected event"),
+       }
+
        connect_blocks(&nodes[2], ANTI_REORG_DELAY - 1);
        check_closed_broadcast!(nodes[2], true);
-       expect_pending_htlcs_forwardable!(nodes[2]);
+       if deliver_last_raa {
+               expect_pending_htlcs_forwardable_from_events!(nodes[2], events[0..1], true);
+       } else {
+               expect_pending_htlcs_forwardable!(nodes[2]);
+       }
        check_added_monitors!(nodes[2], 3);
 
        let cs_msgs = nodes[2].node.get_and_clear_pending_msg_events();
@@ -5453,14 +5516,18 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        let as_events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(as_events.len(), if announce_latest { 5 } else { 3 });
        let mut as_failds = HashSet::new();
+       let mut as_updates = 0;
        for event in as_events.iter() {
-               if let &Event::PaymentFailed { ref payment_hash, ref rejected_by_dest, .. } = event {
+               if let &Event::PaymentPathFailed { ref payment_hash, ref rejected_by_dest, ref network_update, .. } = event {
                        assert!(as_failds.insert(*payment_hash));
                        if *payment_hash != payment_hash_2 {
                                assert_eq!(*rejected_by_dest, deliver_last_raa);
                        } else {
                                assert!(!rejected_by_dest);
                        }
+                       if network_update.is_some() {
+                               as_updates += 1;
+                       }
                } else { panic!("Unexpected event"); }
        }
        assert!(as_failds.contains(&payment_hash_1));
@@ -5474,14 +5541,18 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        let bs_events = nodes[1].node.get_and_clear_pending_events();
        assert_eq!(bs_events.len(), if announce_latest { 4 } else { 3 });
        let mut bs_failds = HashSet::new();
+       let mut bs_updates = 0;
        for event in bs_events.iter() {
-               if let &Event::PaymentFailed { ref payment_hash, ref rejected_by_dest, .. } = event {
+               if let &Event::PaymentPathFailed { ref payment_hash, ref rejected_by_dest, ref network_update, .. } = event {
                        assert!(bs_failds.insert(*payment_hash));
                        if *payment_hash != payment_hash_1 && *payment_hash != payment_hash_5 {
                                assert_eq!(*rejected_by_dest, deliver_last_raa);
                        } else {
                                assert!(!rejected_by_dest);
                        }
+                       if network_update.is_some() {
+                               bs_updates += 1;
+                       }
                } else { panic!("Unexpected event"); }
        }
        assert!(bs_failds.contains(&payment_hash_1));
@@ -5492,20 +5563,11 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno
        assert!(bs_failds.contains(&payment_hash_5));
 
        // For each HTLC which was not failed-back by normal process (ie deliver_last_raa), we should
-       // get a PaymentFailureNetworkUpdate. A should have gotten 4 HTLCs which were failed-back due
-       // to unknown-preimage-etc, B should have gotten 2. Thus, in the
-       // announce_latest && deliver_last_raa case, we should have 5-4=1 and 4-2=2
-       // PaymentFailureNetworkUpdates.
-       let as_msg_events = nodes[0].node.get_and_clear_pending_msg_events();
-       assert_eq!(as_msg_events.len(), if deliver_last_raa { 1 } else if !announce_latest { 3 } else { 5 });
-       let bs_msg_events = nodes[1].node.get_and_clear_pending_msg_events();
-       assert_eq!(bs_msg_events.len(), if deliver_last_raa { 2 } else if !announce_latest { 3 } else { 4 });
-       for event in as_msg_events.iter().chain(bs_msg_events.iter()) {
-               match event {
-                       &MessageSendEvent::PaymentFailureNetworkUpdate { .. } => {},
-                       _ => panic!("Unexpected event"),
-               }
-       }
+       // get a NetworkUpdate. A should have gotten 4 HTLCs which were failed-back due to
+       // unknown-preimage-etc, B should have gotten 2. Thus, in the
+       // announce_latest && deliver_last_raa case, we should have 5-4=1 and 4-2=2 NetworkUpdates.
+       assert_eq!(as_updates, if deliver_last_raa { 1 } else if !announce_latest { 3 } else { 5 });
+       assert_eq!(bs_updates, if deliver_last_raa { 2 } else if !announce_latest { 3 } else { 4 });
 }
 
 #[test]
@@ -5544,6 +5606,7 @@ fn test_dynamic_spendable_outputs_local_htlc_timeout_tx() {
        mine_transaction(&nodes[0], &local_txn[0]);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
 
        let htlc_timeout = {
@@ -5627,6 +5690,7 @@ fn test_key_derivation_params() {
        connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
 
        let htlc_timeout = {
                let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -5667,6 +5731,7 @@ fn test_static_output_closing_tx() {
        let closing_tx = close_channel(&nodes[0], &nodes[1], &chan.2, chan.3, true).2;
 
        mine_transaction(&nodes[0], &closing_tx);
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
        connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
 
        let spend_txn = check_spendable_outputs!(nodes[0], node_cfgs[0].keys_manager);
@@ -5674,6 +5739,7 @@ fn test_static_output_closing_tx() {
        check_spends!(spend_txn[0], closing_tx);
 
        mine_transaction(&nodes[1], &closing_tx);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
 
        let spend_txn = check_spendable_outputs!(nodes[1], node_cfgs[1].keys_manager);
@@ -5724,6 +5790,7 @@ fn do_htlc_claim_local_commitment_only(use_dust: bool) {
        test_txn_broadcast(&nodes[1], &chan, None, if use_dust { HTLCType::NONE } else { HTLCType::SUCCESS });
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 }
 
 fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
@@ -5736,7 +5803,7 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
 
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), if use_dust { 50000 } else { 3000000 }, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), if use_dust { 50000 } else { 3000000 }, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
@@ -5756,6 +5823,7 @@ fn do_htlc_claim_current_remote_commitment_only(use_dust: bool) {
        test_txn_broadcast(&nodes[0], &chan, None, HTLCType::NONE);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
 }
 
 fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no_close: bool) {
@@ -5804,6 +5872,7 @@ fn do_htlc_claim_previous_remote_commitment_only(use_dust: bool, check_revoke_no
                test_txn_broadcast(&nodes[0], &chan, None, HTLCType::NONE);
                check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        } else {
                expect_payment_failed!(nodes[0], our_payment_hash, true);
        }
@@ -5916,7 +5985,7 @@ fn bolt2_open_channel_sane_dust_limit() {
        let push_msat=10001;
        nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), channel_value_satoshis, push_msat, 42, None).unwrap();
        let mut node0_to_1_send_open_channel = get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id());
-       node0_to_1_send_open_channel.dust_limit_satoshis = 661;
+       node0_to_1_send_open_channel.dust_limit_satoshis = 547;
        node0_to_1_send_open_channel.channel_reserve_satoshis = 100001;
 
        nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &node0_to_1_send_open_channel);
@@ -5927,7 +5996,7 @@ fn bolt2_open_channel_sane_dust_limit() {
                },
                _ => panic!("Unexpected event"),
        };
-       assert_eq!(err_msg.data, "dust_limit_satoshis (661) is greater than the implementation limit (660)");
+       assert_eq!(err_msg.data, "dust_limit_satoshis (547) is greater than the implementation limit (546)");
 }
 
 // Test that if we fail to send an HTLC that is being freed from the holding cell, and the HTLC
@@ -5971,7 +6040,7 @@ fn test_fail_holding_cell_htlc_upon_free() {
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let max_can_send = 5000000 - channel_reserve - 2*commit_tx_fee_msat(feerate, 1 + 1);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
 
        // Send a payment which passes reserve checks but gets stuck in the holding cell.
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
@@ -5999,9 +6068,11 @@ fn test_fail_holding_cell_htlc_upon_free() {
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match &events[0] {
-               &Event::PaymentFailed { ref payment_hash, ref rejected_by_dest, ref error_code, ref error_data } => {
+               &Event::PaymentPathFailed { ref payment_hash, ref rejected_by_dest, ref network_update, ref error_code, ref error_data, ref all_paths_failed, path: _ } => {
                        assert_eq!(our_payment_hash.clone(), *payment_hash);
                        assert_eq!(*rejected_by_dest, false);
+                       assert_eq!(*all_paths_failed, true);
+                       assert_eq!(*network_update, None);
                        assert_eq!(*error_code, None);
                        assert_eq!(*error_data, None);
                },
@@ -6051,8 +6122,8 @@ fn test_free_and_fail_holding_cell_htlcs() {
        let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
        let amt_2 = 5000000 - channel_reserve - 2*commit_tx_fee_msat(feerate, 2 + 1) - amt_1;
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route_1 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], amt_1, TEST_FINAL_CLTV, &logger).unwrap();
-       let route_2 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], amt_2, TEST_FINAL_CLTV, &logger).unwrap();
+       let route_1 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], amt_1, TEST_FINAL_CLTV, &logger).unwrap();
+       let route_2 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], amt_2, TEST_FINAL_CLTV, &logger).unwrap();
 
        // Send 2 payments which pass reserve checks but get stuck in the holding cell.
        nodes[0].node.send_payment(&route_1, payment_hash_1, &Some(payment_secret_1)).unwrap();
@@ -6084,9 +6155,11 @@ fn test_free_and_fail_holding_cell_htlcs() {
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
        match &events[0] {
-               &Event::PaymentFailed { ref payment_hash, ref rejected_by_dest, ref error_code, ref error_data } => {
+               &Event::PaymentPathFailed { ref payment_hash, ref rejected_by_dest, ref network_update, ref error_code, ref error_data, ref all_paths_failed, path: _ } => {
                        assert_eq!(payment_hash_2.clone(), *payment_hash);
                        assert_eq!(*rejected_by_dest, false);
+                       assert_eq!(*all_paths_failed, true);
+                       assert_eq!(*network_update, None);
                        assert_eq!(*error_code, None);
                        assert_eq!(*error_data, None);
                },
@@ -6184,7 +6257,7 @@ fn test_fail_holding_cell_htlc_upon_free_multihop() {
        let max_can_send = 5000000 - channel_reserve - 2*commit_tx_fee_msat(feerate, 1 + 1) - total_routing_fee_msat;
        let payment_event = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
 
@@ -6267,8 +6340,7 @@ fn test_fail_holding_cell_htlc_upon_free_multihop() {
                _ => panic!("Unexpected event"),
        };
        nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa);
-       expect_payment_failure_chan_update!(nodes[0], chan_1_2.0.contents.short_channel_id, false);
-       expect_payment_failed!(nodes[0], our_payment_hash, false);
+       expect_payment_failed_with_update!(nodes[0], our_payment_hash, false, chan_1_2.0.contents.short_channel_id, false);
        check_added_monitors!(nodes[0], 1);
 }
 
@@ -6288,7 +6360,7 @@ fn test_update_add_htlc_bolt2_sender_value_below_minimum_msat() {
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        route.paths[0][0].fee_msat = 100;
 
        unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
@@ -6309,7 +6381,7 @@ fn test_update_add_htlc_bolt2_sender_zero_value_msat() {
 
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        route.paths[0][0].fee_msat = 0;
        unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
                assert_eq!(err, "Cannot send 0-msat HTLC"));
@@ -6330,7 +6402,7 @@ fn test_update_add_htlc_bolt2_receiver_zero_value_msat() {
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6340,6 +6412,7 @@ fn test_update_add_htlc_bolt2_receiver_zero_value_msat() {
        nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Remote side tried to send a 0-msat HTLC".to_string(), 1);
        check_closed_broadcast!(nodes[1], true).unwrap();
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "Remote side tried to send a 0-msat HTLC".to_string() });
 }
 
 #[test]
@@ -6356,7 +6429,7 @@ fn test_update_add_htlc_bolt2_sender_cltv_expiry_too_high() {
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
 
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000000, 500000001, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000000, 500000001, &logger).unwrap();
        unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::RouteError { ref err },
                assert_eq!(err, &"Channel CLTV overflowed?"));
 }
@@ -6378,7 +6451,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
                let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
                let payment_event = {
                        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-                       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+                       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
                        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
                        check_added_monitors!(nodes[0], 1);
 
@@ -6400,7 +6473,7 @@ fn test_update_add_htlc_bolt2_sender_exceed_max_htlc_num_and_htlc_id_increment()
        }
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        unwrap_send_err!(nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)), true, APIError::ChannelUnavailable { ref err },
                assert!(regex::Regex::new(r"Cannot push more than their max accepted HTLCs \(\d+\)").unwrap().is_match(err)));
 
@@ -6457,7 +6530,7 @@ fn test_update_add_htlc_bolt2_receiver_check_amount_received_more_than_min() {
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], htlc_minimum_msat, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], htlc_minimum_msat, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6467,6 +6540,7 @@ fn test_update_add_htlc_bolt2_receiver_check_amount_received_more_than_min() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert!(regex::Regex::new(r"Remote side tried to send less than our minimum HTLC value\. Lower limit: \(\d+\)\. Actual: \(\d+\)").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6488,7 +6562,7 @@ fn test_update_add_htlc_bolt2_receiver_sender_can_afford_amount_sent() {
        let max_can_send = 5000000 - channel_reserve - commit_tx_fee_outbound;
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], max_can_send, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6503,6 +6577,7 @@ fn test_update_add_htlc_bolt2_receiver_sender_can_afford_amount_sent() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert_eq!(err_msg.data, "Remote HTLC add would put them under remote reserve value");
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6520,7 +6595,7 @@ fn test_update_add_htlc_bolt2_receiver_check_max_htlc_limit() {
        let session_priv = SecretKey::from_slice(&[42; 32]).unwrap();
 
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 3999999, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 3999999, TEST_FINAL_CLTV, &logger).unwrap();
 
        let cur_height = nodes[0].node.best_block.read().unwrap().height() + 1;
        let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::signing_only(), &route.paths[0], &session_priv).unwrap();
@@ -6547,6 +6622,7 @@ fn test_update_add_htlc_bolt2_receiver_check_max_htlc_limit() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert!(regex::Regex::new(r"Remote tried to push more than our max accepted HTLCs \(\d+\)").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6561,7 +6637,7 @@ fn test_update_add_htlc_bolt2_receiver_check_max_in_flight_msat() {
 
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6572,6 +6648,7 @@ fn test_update_add_htlc_bolt2_receiver_check_max_in_flight_msat() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert!(regex::Regex::new("Remote HTLC add would put them over our max HTLC value").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6586,7 +6663,7 @@ fn test_update_add_htlc_bolt2_receiver_check_cltv_expiry() {
        create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 95000000, InitFeatures::known(), InitFeatures::known());
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let mut updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6597,6 +6674,7 @@ fn test_update_add_htlc_bolt2_receiver_check_cltv_expiry() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert_eq!(err_msg.data,"Remote provided CLTV expiry in seconds instead of block height");
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6613,7 +6691,7 @@ fn test_update_add_htlc_bolt2_receiver_check_repeated_id_ignore() {
        create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6646,6 +6724,7 @@ fn test_update_add_htlc_bolt2_receiver_check_repeated_id_ignore() {
        let err_msg = check_closed_broadcast!(nodes[1], true).unwrap();
        assert!(regex::Regex::new(r"Remote skipped HTLC ID \(skipped ID: \d+\)").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6660,7 +6739,7 @@ fn test_update_fulfill_htlc_bolt2_update_fulfill_htlc_before_commitment() {
        let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
 
        check_added_monitors!(nodes[0], 1);
@@ -6679,6 +6758,7 @@ fn test_update_fulfill_htlc_bolt2_update_fulfill_htlc_before_commitment() {
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert!(regex::Regex::new(r"Remote tried to fulfill/fail HTLC \(\d+\) before it had been committed").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6694,7 +6774,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_htlc_before_commitment() {
 
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6712,6 +6792,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_htlc_before_commitment() {
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert!(regex::Regex::new(r"Remote tried to fulfill/fail HTLC \(\d+\) before it had been committed").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6727,7 +6808,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_malformed_htlc_before_commitment()
 
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -6745,6 +6826,7 @@ fn test_update_fulfill_htlc_bolt2_update_fail_malformed_htlc_before_commitment()
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert!(regex::Regex::new(r"Remote tried to fulfill/fail HTLC \(\d+\) before it had been committed").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6786,6 +6868,7 @@ fn test_update_fulfill_htlc_bolt2_incorrect_htlc_id() {
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert_eq!(err_msg.data, "Remote tried to fulfill/fail an HTLC we couldn't find");
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6827,6 +6910,7 @@ fn test_update_fulfill_htlc_bolt2_wrong_preimage() {
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert!(regex::Regex::new(r"Remote tried to fulfill HTLC \(\d+\) with an incorrect preimage").unwrap().is_match(err_msg.data.as_str()));
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6842,7 +6926,7 @@ fn test_update_fulfill_htlc_bolt2_missing_badonion_bit_for_malformed_htlc_messag
 
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 1000000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
 
@@ -6875,6 +6959,7 @@ fn test_update_fulfill_htlc_bolt2_missing_badonion_bit_for_malformed_htlc_messag
        let err_msg = check_closed_broadcast!(nodes[0], true).unwrap();
        assert_eq!(err_msg.data, "Got update_fail_malformed_htlc with BADONION not set");
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: err_msg.data });
 }
 
 #[test]
@@ -6895,7 +6980,7 @@ fn test_update_fulfill_htlc_bolt2_after_malformed_htlc_message_must_forward_upda
        //First hop
        let mut payment_event = {
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 100000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -7017,16 +7102,17 @@ fn do_test_failure_delay_dust_htlc_local_commitment(announce_latest: bool) {
 
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
 
        assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
        connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
        let events = nodes[0].node.get_and_clear_pending_events();
-       // Only 2 PaymentFailed events should show up, over-dust HTLC has to be failed by timeout tx
+       // Only 2 PaymentPathFailed events should show up, over-dust HTLC has to be failed by timeout tx
        assert_eq!(events.len(), 2);
        let mut first_failed = false;
        for event in events {
                match event {
-                       Event::PaymentFailed { payment_hash, .. } => {
+                       Event::PaymentPathFailed { payment_hash, .. } => {
                                if payment_hash == payment_hash_1 {
                                        assert!(!first_failed);
                                        first_failed = true;
@@ -7077,6 +7163,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
        if local {
                // We fail dust-HTLC 1 by broadcast of local commitment tx
                mine_transaction(&nodes[0], &as_commitment_tx[0]);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
                expect_payment_failed!(nodes[0], dust_hash, true);
 
@@ -7096,6 +7183,7 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
                mine_transaction(&nodes[0], &bs_commitment_tx[0]);
                check_closed_broadcast!(nodes[0], true);
                check_added_monitors!(nodes[0], 1);
+               check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
                assert_eq!(nodes[0].node.get_and_clear_pending_events().len(), 0);
                connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1); // Confirm blocks until the HTLC expires
                timeout_tx.push(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[1].clone());
@@ -7114,14 +7202,14 @@ fn do_test_sweep_outbound_htlc_failure_update(revoked: bool, local: bool) {
                        assert_eq!(events.len(), 2);
                        let first;
                        match events[0] {
-                               Event::PaymentFailed { payment_hash, .. } => {
+                               Event::PaymentPathFailed { payment_hash, .. } => {
                                        if payment_hash == dust_hash { first = true; }
                                        else { first = false; }
                                },
                                _ => panic!("Unexpected event"),
                        }
                        match events[1] {
-                               Event::PaymentFailed { payment_hash, .. } => {
+                               Event::PaymentPathFailed { payment_hash, .. } => {
                                        if first { assert_eq!(payment_hash, non_dust_hash); }
                                        else { assert_eq!(payment_hash, dust_hash); }
                                },
@@ -7177,14 +7265,17 @@ fn test_user_configurable_csv_delay() {
        let mut accept_channel = get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id());
        accept_channel.to_self_delay = 200;
        nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &accept_channel);
+       let reason_msg;
        if let MessageSendEvent::HandleError { ref action, .. } = nodes[0].node.get_and_clear_pending_msg_events()[0] {
                match action {
                        &ErrorAction::SendErrorMessage { ref msg } => {
                                assert!(regex::Regex::new(r"They wanted our payments to be delayed by a needlessly long period\. Upper limit: \d+\. Actual: \d+").unwrap().is_match(msg.data.as_str()));
+                               reason_msg = msg.data.clone();
                        },
-                       _ => { assert!(false); }
+                       _ => { panic!(); }
                }
-       } else { assert!(false); }
+       } else { panic!(); }
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: reason_msg });
 
        // We test msg.to_self_delay <= config.their_to_self_delay is enforced in Channel::new_from_req()
        nodes[1].node.create_channel(nodes[0].node.get_our_node_id(), 1000000, 1000000, 42, None).unwrap();
@@ -7297,10 +7388,10 @@ fn test_data_loss_protect() {
 
        // Check we close channel detecting A is fallen-behind
        nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "Peer attempted to reestablish channel with a very old local commitment transaction".to_string() });
        assert_eq!(check_closed_broadcast!(nodes[1], true).unwrap().data, "Peer attempted to reestablish channel with a very old local commitment transaction");
        check_added_monitors!(nodes[1], 1);
 
-
        // Check A is able to claim to_remote output
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
        assert_eq!(node_txn.len(), 1);
@@ -7308,6 +7399,7 @@ fn test_data_loss_protect() {
        assert_eq!(node_txn[0].output.len(), 2);
        mine_transaction(&nodes[0], &node_txn[0]);
        connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "We have fallen behind - we have received proof that if we broadcast remote is going to claim our funds - we can\'t do any automated broadcasting".to_string() });
        let spend_txn = check_spendable_outputs!(nodes[0], node_cfgs[0].keys_manager);
        assert_eq!(spend_txn.len(), 1);
        check_spends!(spend_txn[0], node_txn[0]);
@@ -7327,7 +7419,7 @@ fn test_check_htlc_underpaying() {
        // Create some initial channels
        create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
 
-       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 10_000, TEST_FINAL_CLTV, nodes[0].logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 10_000, TEST_FINAL_CLTV, nodes[0].logger).unwrap();
        let (_, our_payment_hash, _) = get_payment_preimage_hash!(nodes[0]);
        let our_payment_secret = nodes[1].node.create_inbound_payment_for_hash(our_payment_hash, Some(100_000), 7200, 0).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
@@ -7508,7 +7600,7 @@ fn test_priv_forwarding_rejection() {
        // to nodes[2], which should be rejected:
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[2]);
        let route = get_route(&nodes[0].node.get_our_node_id(),
-               &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(),
+               &nodes[0].net_graph_msg_handler.network_graph,
                &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None,
                &[&RouteHint(vec![RouteHintHop {
                        src_node_id: nodes[1].node.get_our_node_id(),
@@ -7533,8 +7625,7 @@ fn test_priv_forwarding_rejection() {
 
        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &htlc_fail_updates.update_fail_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], htlc_fail_updates.commitment_signed, true, true);
-       expect_payment_failed!(nodes[0], our_payment_hash, false);
-       expect_payment_failure_chan_update!(nodes[0], nodes[2].node.list_channels()[0].short_channel_id.unwrap(), true);
+       expect_payment_failed_with_update!(nodes[0], our_payment_hash, false, nodes[2].node.list_channels()[0].short_channel_id.unwrap(), true);
 
        // Now disconnect nodes[1] from its peers and restart with accept_forwards_to_priv_channels set
        // to true. Sadly there is currently no way to change it at runtime.
@@ -7628,7 +7719,7 @@ fn test_bump_penalty_txn_on_revoked_commitment() {
 
        let payment_preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 3000000).0;
        let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3000000, 30, &logger).unwrap();
+       let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3000000, 30, &logger).unwrap();
        send_along_route(&nodes[1], route, &vec!(&nodes[0])[..], 3000000);
 
        let revoked_txn = get_local_commitment_txn!(nodes[0], chan.2);
@@ -7732,10 +7823,10 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
 
        let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 59000000, InitFeatures::known(), InitFeatures::known());
        // Lock HTLC in both directions (using a slightly lower CLTV delay to provide timely RBF bumps)
-       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph,
                &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3_000_000, 50, nodes[0].logger).unwrap();
        let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 3_000_000).0;
-       let route = get_route(&nodes[1].node.get_our_node_id(), &nodes[1].net_graph_msg_handler.network_graph.read().unwrap(),
+       let route = get_route(&nodes[1].node.get_our_node_id(), &nodes[1].net_graph_msg_handler.network_graph,
                &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3_000_000, 50, nodes[0].logger).unwrap();
        send_along_route(&nodes[1], route, &[&nodes[0]], 3_000_000);
 
@@ -7751,6 +7842,7 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        connect_block(&nodes[1], &Block { header, txdata: vec![revoked_local_txn[0].clone()] });
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[1], 49); // Confirm blocks until the HTLC expires (note CLTV was explicitly 50 above)
 
        let revoked_htlc_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
@@ -7772,7 +7864,12 @@ fn test_bump_penalty_txn_on_revoked_htlcs() {
        connect_block(&nodes[0], &Block { header: header_11, txdata: vec![revoked_local_txn[0].clone()] });
        let header_129 = BlockHeader { version: 0x20000000, prev_blockhash: header_11.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
        connect_block(&nodes[0], &Block { header: header_129, txdata: vec![revoked_htlc_txn[0].clone(), revoked_htlc_txn[2].clone()] });
-       expect_pending_htlcs_forwardable_ignore!(nodes[0]);
+       let events = nodes[0].node.get_and_clear_pending_events();
+       expect_pending_htlcs_forwardable_from_events!(nodes[0], events[0..1], true);
+       match events[1] {
+               Event::ChannelClosed { reason: ClosureReason::CommitmentTxConfirmed, .. } => {}
+               _ => panic!("Unexpected event"),
+       }
        let first;
        let feerate_1;
        let penalty_txn;
@@ -8018,6 +8115,7 @@ fn test_counterparty_raa_skip_no_crash() {
                &msgs::RevokeAndACK { channel_id, per_commitment_secret, next_per_commitment_point });
        assert_eq!(check_closed_broadcast!(nodes[1], true).unwrap().data, "Received an unexpected revoke_and_ack");
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "Received an unexpected revoke_and_ack".to_string() });
 }
 
 #[test]
@@ -8050,6 +8148,7 @@ fn test_bump_txn_sanitize_tracking_maps() {
        mine_transaction(&nodes[0], &revoked_local_txn[0]);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        let penalty_txn = {
                let mut node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
                assert_eq!(node_txn.len(), 4); //ChannelMonitor: justice txn * 3, ChannelManager: local commitment tx
@@ -8125,7 +8224,7 @@ fn test_simple_mpp() {
 
        let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[3]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let mut route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        let path = route.paths[0].clone();
        route.paths.push(path);
        route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
@@ -8153,7 +8252,7 @@ fn test_preimage_storage() {
 
                let logger = test_utils::TestLogger::new();
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -8224,7 +8323,7 @@ fn test_secret_timeout() {
        {
                let logger = test_utils::TestLogger::new();
                let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
                nodes[0].node.send_payment(&route, payment_hash, &Some(our_payment_secret)).unwrap();
                check_added_monitors!(nodes[0], 1);
                let mut events = nodes[0].node.get_and_clear_pending_msg_events();
@@ -8264,7 +8363,7 @@ fn test_bad_secret_hash() {
 
        let logger = test_utils::TestLogger::new();
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100_000, TEST_FINAL_CLTV, &logger).unwrap();
 
        // All the below cases should end up being handled exactly identically, so we macro the
        // resulting events.
@@ -8453,7 +8552,7 @@ fn test_concurrent_monitor_claim() {
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[0]);
        {
                let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
-               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3000000 , TEST_FINAL_CLTV, &logger).unwrap();
+               let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 3000000 , TEST_FINAL_CLTV, &logger).unwrap();
                nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
        }
        check_added_monitors!(nodes[1], 1);
@@ -8533,6 +8632,7 @@ fn test_pre_lockin_no_chan_closed_update() {
        let channel_id = ::chain::transaction::OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
        nodes[0].node.handle_error(&nodes[1].node.get_our_node_id(), &msgs::ErrorMessage { channel_id, data: "Hi".to_owned() });
        assert!(nodes[0].chain_monitor.added_monitors.lock().unwrap().is_empty());
+       check_closed_event!(nodes[0], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "Hi".to_string() });
 }
 
 #[test]
@@ -8567,6 +8667,7 @@ fn test_htlc_no_detection() {
        chain::Listen::block_connected(&nodes[0].chain_monitor.chain_monitor, &Block { header, txdata: vec![local_txn[0].clone()] }, nodes[0].best_block_info().1 + 1);
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        connect_blocks(&nodes[0], TEST_FINAL_CLTV - 1);
 
        let htlc_timeout = {
@@ -8626,6 +8727,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
        nodes[force_closing_node].node.force_close_channel(&chan_ab.2).unwrap();
        check_closed_broadcast!(nodes[force_closing_node], true);
        check_added_monitors!(nodes[force_closing_node], 1);
+       check_closed_event!(nodes[force_closing_node], 1, ClosureReason::HolderForceClosed);
        if go_onchain_before_fulfill {
                let txn_to_broadcast = match broadcast_alice {
                        true => alice_txn.clone(),
@@ -8637,6 +8739,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
                if broadcast_alice {
                        check_closed_broadcast!(nodes[1], true);
                        check_added_monitors!(nodes[1], 1);
+                       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                }
                assert_eq!(bob_txn.len(), 1);
                check_spends!(bob_txn[0], chan_ab.3);
@@ -8717,6 +8820,7 @@ fn do_test_onchain_htlc_settlement_after_close(broadcast_alice: bool, go_onchain
                if broadcast_alice {
                        check_closed_broadcast!(nodes[1], true);
                        check_added_monitors!(nodes[1], 1);
+                       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
                }
                let mut bob_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
                if broadcast_alice {
@@ -8932,6 +9036,7 @@ fn test_error_chans_closed() {
        nodes[0].node.handle_error(&nodes[1].node.get_our_node_id(), &msgs::ErrorMessage { channel_id: chan_2.2, data: "ERR".to_owned() });
        check_added_monitors!(nodes[0], 1);
        check_closed_broadcast!(nodes[0], false);
+       check_closed_event!(nodes[0], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "ERR".to_string() });
        assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0).len(), 1);
        assert_eq!(nodes[0].node.list_usable_channels().len(), 2);
        assert!(nodes[0].node.list_usable_channels()[0].channel_id == chan_1.2 || nodes[0].node.list_usable_channels()[1].channel_id == chan_1.2);
@@ -8941,6 +9046,7 @@ fn test_error_chans_closed() {
        let _chan_4 = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 10001, InitFeatures::known(), InitFeatures::known());
        nodes[0].node.handle_error(&nodes[1].node.get_our_node_id(), &msgs::ErrorMessage { channel_id: [0; 32], data: "ERR".to_owned() });
        check_added_monitors!(nodes[0], 2);
+       check_closed_event!(nodes[0], 2, ClosureReason::CounterpartyForceClosed { peer_msg: "ERR".to_string() });
        let events = nodes[0].node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), 2);
        match events[0] {
@@ -9005,6 +9111,7 @@ fn test_invalid_funding_tx() {
        nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 
        confirm_transaction_at(&nodes[1], &tx, 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        check_added_monitors!(nodes[1], 1);
        let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
        assert_eq!(events_2.len(), 1);
@@ -9048,6 +9155,7 @@ fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_t
 
        nodes[1].node.force_close_channel(&channel_id).unwrap();
        check_closed_broadcast!(nodes[1], true);
+       check_closed_event!(nodes[1], 1, ClosureReason::HolderForceClosed);
        check_added_monitors!(nodes[1], 1);
        let node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
        assert_eq!(node_txn.len(), 1);
@@ -9090,8 +9198,7 @@ fn do_test_tx_confirmed_skipping_blocks_immediate_broadcast(test_height_before_t
                assert!(updates.update_fee.is_none());
                nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
                commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, true, true);
-               expect_payment_failed!(nodes[0], payment_hash, false);
-               expect_payment_failure_chan_update!(nodes[0], chan_announce.contents.short_channel_id, true);
+               expect_payment_failed_with_update!(nodes[0], payment_hash, false, chan_announce.contents.short_channel_id, true);
        }
 }
 
@@ -9101,6 +9208,125 @@ fn test_tx_confirmed_skipping_blocks_immediate_broadcast() {
        do_test_tx_confirmed_skipping_blocks_immediate_broadcast(true);
 }
 
+#[test]
+fn test_forwardable_regen() {
+       // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
+       // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
+       // HTLCs.
+       // We test it for both payment receipt and payment forwarding.
+
+       let chanmon_cfgs = create_chanmon_cfgs(3);
+       let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
+       let persister: test_utils::TestPersister;
+       let new_chain_monitor: test_utils::TestChainMonitor;
+       let nodes_1_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
+       let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
+       create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
+       create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
+
+       // First send a payment to nodes[1]
+       let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
+       nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
+       check_added_monitors!(nodes[0], 1);
+
+       let mut events = nodes[0].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       let payment_event = SendEvent::from_event(events.pop().unwrap());
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
+
+       expect_pending_htlcs_forwardable_ignore!(nodes[1]);
+
+       // Next send a payment which is forwarded by nodes[1]
+       let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
+       nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2)).unwrap();
+       check_added_monitors!(nodes[0], 1);
+
+       let mut events = nodes[0].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       let payment_event = SendEvent::from_event(events.pop().unwrap());
+       nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
+       commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
+
+       // There is already a PendingHTLCsForwardable event "pending" so another one will not be
+       // generated
+       assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
+
+       // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
+       nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
+       nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
+
+       let nodes_1_serialized = nodes[1].node.encode();
+       let mut chan_0_monitor_serialized = test_utils::TestVecWriter(Vec::new());
+       let mut chan_1_monitor_serialized = test_utils::TestVecWriter(Vec::new());
+       {
+               let monitors = nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap();
+               let mut monitor_iter = monitors.iter();
+               monitor_iter.next().unwrap().1.write(&mut chan_0_monitor_serialized).unwrap();
+               monitor_iter.next().unwrap().1.write(&mut chan_1_monitor_serialized).unwrap();
+       }
+
+       persister = test_utils::TestPersister::new();
+       let keys_manager = &chanmon_cfgs[1].keys_manager;
+       new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[1].chain_source), nodes[1].tx_broadcaster.clone(), nodes[1].logger, node_cfgs[1].fee_estimator, &persister, keys_manager);
+       nodes[1].chain_monitor = &new_chain_monitor;
+
+       let mut chan_0_monitor_read = &chan_0_monitor_serialized.0[..];
+       let (_, mut chan_0_monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
+               &mut chan_0_monitor_read, keys_manager).unwrap();
+       assert!(chan_0_monitor_read.is_empty());
+       let mut chan_1_monitor_read = &chan_1_monitor_serialized.0[..];
+       let (_, mut chan_1_monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
+               &mut chan_1_monitor_read, keys_manager).unwrap();
+       assert!(chan_1_monitor_read.is_empty());
+
+       let mut nodes_1_read = &nodes_1_serialized[..];
+       let (_, nodes_1_deserialized_tmp) = {
+               let mut channel_monitors = HashMap::new();
+               channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
+               channel_monitors.insert(chan_1_monitor.get_funding_txo().0, &mut chan_1_monitor);
+               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_1_read, ChannelManagerReadArgs {
+                       default_config: UserConfig::default(),
+                       keys_manager,
+                       fee_estimator: node_cfgs[1].fee_estimator,
+                       chain_monitor: nodes[1].chain_monitor,
+                       tx_broadcaster: nodes[1].tx_broadcaster.clone(),
+                       logger: nodes[1].logger,
+                       channel_monitors,
+               }).unwrap()
+       };
+       nodes_1_deserialized = nodes_1_deserialized_tmp;
+       assert!(nodes_1_read.is_empty());
+
+       assert!(nodes[1].chain_monitor.watch_channel(chan_0_monitor.get_funding_txo().0, chan_0_monitor).is_ok());
+       assert!(nodes[1].chain_monitor.watch_channel(chan_1_monitor.get_funding_txo().0, chan_1_monitor).is_ok());
+       nodes[1].node = &nodes_1_deserialized;
+       check_added_monitors!(nodes[1], 2);
+
+       reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+       // Note that nodes[1] and nodes[2] resend their funding_locked here since they haven't updated
+       // the commitment state.
+       reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
+
+       assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
+
+       expect_pending_htlcs_forwardable!(nodes[1]);
+       expect_payment_received!(nodes[1], payment_hash, payment_secret, 100_000);
+       check_added_monitors!(nodes[1], 1);
+
+       let mut events = nodes[1].node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), 1);
+       let payment_event = SendEvent::from_event(events.pop().unwrap());
+       nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
+       commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
+       expect_pending_htlcs_forwardable!(nodes[2]);
+       expect_payment_received!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
+
+       claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
+       claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
+}
+
 #[test]
 fn test_keysend_payments_to_public_node() {
        let chanmon_cfgs = create_chanmon_cfgs(2);
@@ -9109,10 +9335,10 @@ fn test_keysend_payments_to_public_node() {
        let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 
        let _chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100000, 10001, InitFeatures::known(), InitFeatures::known());
-       let network_graph = nodes[0].net_graph_msg_handler.network_graph.read().unwrap();
+       let network_graph = &nodes[0].net_graph_msg_handler.network_graph;
        let payer_pubkey = nodes[0].node.get_our_node_id();
        let payee_pubkey = nodes[1].node.get_our_node_id();
-       let route = get_route(&payer_pubkey, &network_graph, &payee_pubkey, None,
+       let route = get_route(&payer_pubkey, network_graph, &payee_pubkey, None,
                         None, &vec![], 10000, 40,
                         nodes[0].logger).unwrap();
 
@@ -9140,7 +9366,7 @@ fn test_keysend_payments_to_private_node() {
        nodes[1].node.peer_connected(&payer_pubkey, &msgs::Init { features: InitFeatures::known() });
 
        let _chan = create_chan_between_nodes(&nodes[0], &nodes[1], InitFeatures::known(), InitFeatures::known());
-       let network_graph = nodes[0].net_graph_msg_handler.network_graph.read().unwrap();
+       let network_graph = &nodes[0].net_graph_msg_handler.network_graph;
        let first_hops = nodes[0].node.list_usable_channels();
        let route = get_keysend_route(&payer_pubkey, &network_graph, &payee_pubkey,
                                 Some(&first_hops.iter().collect::<Vec<_>>()), &vec![], 10000, 40,
@@ -9156,116 +9382,3 @@ fn test_keysend_payments_to_private_node() {
        pass_along_path(&nodes[0], &path, 10000, payment_hash, None, event, true, Some(test_preimage));
        claim_payment(&nodes[0], &path, test_preimage);
 }
-
-fn do_test_max_dust_htlc_exposure(dust_outbound_balance: bool, at_forward: bool, on_holder_tx: bool) {
-       // Test that we properly reject dust HTLC violating our `max_dust_htlc_exposure_msat` policy.
-       //
-       // At HTLC forward (`send_payment()`), if the sum of the trimmed-to-dust HTLC inbound and
-       // trimmed-to-dust HTLC outbound balance and this new payment as included on next counterparty
-       // commitment are above our `max_dust_htlc_exposure_msat`, we'll reject the update.
-       // At HTLC reception (`update_add_htlc()`), if the sum of the trimmed-to-dust HTLC inbound
-       // and trimmed-to-dust HTLC outbound balance and this new received HTLC as included on next
-       // counterparty commitment are above our `max_dust_htlc_exposure_msat`, we'll fail the update.
-       // Note, we return a `temporary_channel_failure` (0x1000 | 7), as the channel might be
-       // available again for HTLC processing once the dust bandwidth has cleared up.
-
-       let chanmon_cfgs = create_chanmon_cfgs(2);
-       let mut config = test_default_channel_config();
-       config.channel_options.max_dust_htlc_exposure_msat = 5_000_000; // default setting value
-       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
-       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, Some(config)]);
-       let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
-
-       nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 1_000_000, 500_000_000, 42, None).unwrap();
-       let mut open_channel = get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id());
-       open_channel.max_htlc_value_in_flight_msat = 50_000_000;
-       open_channel.max_accepted_htlcs = 60;
-       nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &open_channel);
-       let mut accept_channel = get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id());
-       if on_holder_tx {
-               accept_channel.dust_limit_satoshis = 660;
-       }
-       nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &accept_channel);
-
-       let (temporary_channel_id, tx, _) = create_funding_transaction(&nodes[0], 1_000_000, 42);
-
-       if on_holder_tx {
-               if let Some(mut chan) = nodes[1].node.channel_state.lock().unwrap().by_id.get_mut(&temporary_channel_id) {
-                       chan.holder_dust_limit_satoshis = 660;
-               }
-       }
-
-       nodes[0].node.funding_transaction_generated(&temporary_channel_id, tx.clone()).unwrap();
-       nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
-       check_added_monitors!(nodes[1], 1);
-
-       nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
-       check_added_monitors!(nodes[0], 1);
-
-       let (funding_locked, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
-       let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked);
-       update_nodes_with_chan_announce(&nodes, 0, 1, &announcement, &as_update, &bs_update);
-
-       if on_holder_tx {
-               if dust_outbound_balance {
-                       for i in 0..2 {
-                               let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 2_300_000);
-                               if let Err(_) = nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)) { panic!("Unexpected event at dust HTLC {}", i); }
-                       }
-               } else {
-                       for _ in 0..2 {
-                               route_payment(&nodes[0], &[&nodes[1]], 2_300_000);
-                       }
-               }
-       } else {
-               if dust_outbound_balance {
-                       for i in 0..25 {
-                               let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], 200_000); // + 177_000 msat of HTLC-success tx at 253 sats/kWU
-                               if let Err(_) = nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)) { panic!("Unexpected event at dust HTLC {}", i); }
-                       }
-               } else {
-                       for _ in 0..25 {
-                               route_payment(&nodes[0], &[&nodes[1]], 200_000); // + 167_000 msat of HTLC-timeout tx at 253 sats/kWU
-                       }
-               }
-       }
-
-       if at_forward {
-               let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[1], nodes[0], if on_holder_tx { 2_300_000 } else { 200_000 });
-               let mut config = UserConfig::default();
-               if on_holder_tx {
-                       unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", 6_900_000, config.channel_options.max_dust_htlc_exposure_msat)));
-               } else {
-                       unwrap_send_err!(nodes[1].node.send_payment(&route, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable { ref err }, assert_eq!(err, &format!("Cannot send value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", 5_200_000, config.channel_options.max_dust_htlc_exposure_msat)));
-               }
-       } else {
-               let (route, payment_hash, _, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1 ], if on_holder_tx { 2_300_000 } else { 200_000 });
-               nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)).unwrap();
-               check_added_monitors!(nodes[0], 1);
-               let mut events = nodes[0].node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), 1);
-               let payment_event = SendEvent::from_event(events.remove(0));
-               nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
-               if on_holder_tx {
-                       nodes[1].logger.assert_log("lightning::ln::channel".to_string(), format!("Cannot accept value that would put our exposure to dust HTLCs at {} over the limit {} on holder commitment tx", 6_900_000, config.channel_options.max_dust_htlc_exposure_msat), 1);
-               } else {
-                       nodes[1].logger.assert_log("lightning::ln::channel".to_string(), format!("Cannot accept value that would put our exposure to dust HTLCs at {} over the limit {} on counterparty commitment tx", 5_200_000, config.channel_options.max_dust_htlc_exposure_msat), 1);
-               }
-       }
-
-       let _ = nodes[1].node.get_and_clear_pending_msg_events();
-       let mut added_monitors = nodes[1].chain_monitor.added_monitors.lock().unwrap();
-       added_monitors.clear();
-}
-
-#[test]
-fn test_max_dust_htlc_exposure() {
-       do_test_max_dust_htlc_exposure(true, true, true);
-       do_test_max_dust_htlc_exposure(false, true, true);
-       do_test_max_dust_htlc_exposure(false, false, true);
-       do_test_max_dust_htlc_exposure(false, false, false);
-       do_test_max_dust_htlc_exposure(true, true, false);
-       do_test_max_dust_htlc_exposure(true, false, false);
-       do_test_max_dust_htlc_exposure(true, false, true);
-       do_test_max_dust_htlc_exposure(false, true, false);
-}
index 8fdf28e578b835342c498ba39d4132ec5ab9bd0b..08cd2306de865970d70afa8bc72b85b4d0909ac6 100644 (file)
@@ -9,16 +9,23 @@
 
 //! Further functional tests which test blockchain reorganizations.
 
-use chain::channelmonitor::ANTI_REORG_DELAY;
-use ln::{PaymentPreimage, PaymentHash};
+use chain::channelmonitor::{ANTI_REORG_DELAY, Balance};
+use chain::transaction::OutPoint;
+use ln::{channel, PaymentPreimage, PaymentHash};
+use ln::channelmanager::BREAKDOWN_TIMEOUT;
 use ln::features::InitFeatures;
-use ln::msgs::{ChannelMessageHandler, HTLCFailChannelUpdate, ErrorAction};
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
+use ln::msgs::{ChannelMessageHandler, ErrorAction};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
+use routing::network_graph::NetworkUpdate;
 use routing::router::get_route;
 
 use bitcoin::hashes::sha256::Hash as Sha256;
 use bitcoin::hashes::Hash;
 
+use bitcoin::blockdata::script::Builder;
+use bitcoin::blockdata::opcodes;
+use bitcoin::secp256k1::Secp256k1;
+
 use prelude::*;
 
 use ln::functional_test_utils::*;
@@ -67,6 +74,7 @@ fn chanmon_fail_from_stale_commitment() {
        mine_transaction(&nodes[1], &bs_txn[0]);
        check_added_monitors!(nodes[1], 1);
        check_closed_broadcast!(nodes[1], true);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
@@ -76,6 +84,456 @@ fn chanmon_fail_from_stale_commitment() {
 
        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &fail_updates.update_fail_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], fail_updates.commitment_signed, true, true);
-       expect_payment_failed!(nodes[0], payment_hash, false);
-       expect_payment_failure_chan_update!(nodes[0], update_a.contents.short_channel_id, true);
+       expect_payment_failed_with_update!(nodes[0], payment_hash, false, update_a.contents.short_channel_id, true);
+}
+
+#[test]
+fn chanmon_claim_value_coop_close() {
+       // Tests `get_claimable_balances` returns the correct values across a simple cooperative claim.
+       // Specifically, this tests that the channel non-HTLC balances show up in
+       // `get_claimable_balances` until the cooperative claims have confirmed and generated a
+       // `SpendableOutputs` event, and no longer.
+       let chanmon_cfgs = create_chanmon_cfgs(2);
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, chan_id, funding_tx) =
+               create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 1_000_000, InitFeatures::known(), InitFeatures::known());
+       let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 };
+       assert_eq!(funding_outpoint.to_channel_id(), chan_id);
+
+       let chan_feerate = get_feerate!(nodes[0], chan_id) as u64;
+
+       assert_eq!(vec![Balance::ClaimableOnChannelClose {
+                       claimable_amount_satoshis: 1_000_000 - 1_000 - chan_feerate * channel::COMMITMENT_TX_BASE_WEIGHT / 1000
+               }],
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       assert_eq!(vec![Balance::ClaimableOnChannelClose { claimable_amount_satoshis: 1_000, }],
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       nodes[0].node.close_channel(&chan_id).unwrap();
+       let node_0_shutdown = get_event_msg!(nodes[0], MessageSendEvent::SendShutdown, nodes[1].node.get_our_node_id());
+       nodes[1].node.handle_shutdown(&nodes[0].node.get_our_node_id(), &InitFeatures::known(), &node_0_shutdown);
+       let node_1_shutdown = get_event_msg!(nodes[1], MessageSendEvent::SendShutdown, nodes[0].node.get_our_node_id());
+       nodes[0].node.handle_shutdown(&nodes[1].node.get_our_node_id(), &InitFeatures::known(), &node_1_shutdown);
+
+       let node_0_closing_signed = get_event_msg!(nodes[0], MessageSendEvent::SendClosingSigned, nodes[1].node.get_our_node_id());
+       nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &node_0_closing_signed);
+       let node_1_closing_signed = get_event_msg!(nodes[1], MessageSendEvent::SendClosingSigned, nodes[0].node.get_our_node_id());
+       nodes[0].node.handle_closing_signed(&nodes[1].node.get_our_node_id(), &node_1_closing_signed);
+       let (_, node_0_2nd_closing_signed) = get_closing_signed_broadcast!(nodes[0].node, nodes[1].node.get_our_node_id());
+       nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &node_0_2nd_closing_signed.unwrap());
+       let (_, node_1_none) = get_closing_signed_broadcast!(nodes[1].node, nodes[0].node.get_our_node_id());
+       assert!(node_1_none.is_none());
+
+       let shutdown_tx = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+       assert_eq!(shutdown_tx, nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0));
+       assert_eq!(shutdown_tx.len(), 1);
+
+       mine_transaction(&nodes[0], &shutdown_tx[0]);
+       mine_transaction(&nodes[1], &shutdown_tx[0]);
+
+       assert!(nodes[0].node.list_channels().is_empty());
+       assert!(nodes[1].node.list_channels().is_empty());
+
+       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+
+       assert_eq!(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1_000_000 - 1_000 - chan_feerate * channel::COMMITMENT_TX_BASE_WEIGHT / 1000,
+                       confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1,
+               }],
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       assert_eq!(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1000,
+                       confirmation_height: nodes[1].best_block_info().1 + ANTI_REORG_DELAY - 1,
+               }],
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
+       connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
+
+       assert_eq!(Vec::<Balance>::new(),
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       assert_eq!(Vec::<Balance>::new(),
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       let mut node_a_spendable = nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_a_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_a_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[0].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, shutdown_tx[0]);
+       }
+
+       let mut node_b_spendable = nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_b_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_b_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[1].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, shutdown_tx[0]);
+       }
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+}
+
+fn sorted_vec<T: Ord>(mut v: Vec<T>) -> Vec<T> {
+       v.sort_unstable();
+       v
+}
+
+fn do_test_claim_value_force_close(prev_commitment_tx: bool) {
+       // Tests `get_claimable_balances` with an HTLC across a force-close.
+       // We build a channel with an HTLC pending, then force close the channel and check that the
+       // `get_claimable_balances` return value is correct as transactions confirm on-chain.
+       let mut chanmon_cfgs = create_chanmon_cfgs(2);
+       if prev_commitment_tx {
+               // We broadcast a second-to-latest commitment transaction, without providing the revocation
+               // secret to the counterparty. However, because we always immediately take the revocation
+               // secret from the keys_manager, we would panic at broadcast as we're trying to sign a
+               // transaction which, from the point of view of our keys_manager, is revoked.
+               chanmon_cfgs[1].keys_manager.disable_revocation_policy_check = true;
+       }
+       let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
+       let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
+       let nodes = create_network(2, &node_cfgs, &node_chanmgrs);
+
+       let (_, _, chan_id, funding_tx) =
+               create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1_000_000, 1_000_000, InitFeatures::known(), InitFeatures::known());
+       let funding_outpoint = OutPoint { txid: funding_tx.txid(), index: 0 };
+       assert_eq!(funding_outpoint.to_channel_id(), chan_id);
+
+       // This HTLC is immediately claimed, giving node B the preimage
+       let payment_preimage = route_payment(&nodes[0], &[&nodes[1]], 3_000_000).0;
+       // This HTLC is allowed to time out, letting A claim it. However, in order to test claimable
+       // balances more fully we also give B the preimage for this HTLC.
+       let (timeout_payment_preimage, timeout_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 4_000_000);
+       // This HTLC will be dust, and not be claimable at all:
+       let (dust_payment_preimage, dust_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 3_000);
+
+       let htlc_cltv_timeout = nodes[0].best_block_info().1 + TEST_FINAL_CLTV + 1; // Note ChannelManager adds one to CLTV timeouts for safety
+
+       let chan_feerate = get_feerate!(nodes[0], chan_id) as u64;
+
+       let remote_txn = get_local_commitment_txn!(nodes[1], chan_id);
+       // Before B receives the payment preimage, it only suggests the push_msat value of 1_000 sats
+       // as claimable. A lists both its to-self balance and the (possibly-claimable) HTLCs.
+       assert_eq!(sorted_vec(vec![Balance::ClaimableOnChannelClose {
+                       claimable_amount_satoshis: 1_000_000 - 3_000 - 4_000 - 1_000 - 3 - chan_feerate *
+                               (channel::COMMITMENT_TX_BASE_WEIGHT + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 3_000,
+                       claimable_height: htlc_cltv_timeout,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+       assert_eq!(vec![Balance::ClaimableOnChannelClose {
+                       claimable_amount_satoshis: 1_000,
+               }],
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       nodes[1].node.claim_funds(payment_preimage);
+       check_added_monitors!(nodes[1], 1);
+       let b_htlc_msgs = get_htlc_update_msgs!(&nodes[1], nodes[0].node.get_our_node_id());
+       // We claim the dust payment here as well, but it won't impact our claimable balances as its
+       // dust and thus doesn't appear on chain at all.
+       nodes[1].node.claim_funds(dust_payment_preimage);
+       check_added_monitors!(nodes[1], 1);
+       nodes[1].node.claim_funds(timeout_payment_preimage);
+       check_added_monitors!(nodes[1], 1);
+
+       if prev_commitment_tx {
+               // To build a previous commitment transaction, deliver one round of commitment messages.
+               nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &b_htlc_msgs.update_fulfill_htlcs[0]);
+               expect_payment_sent!(nodes[0], payment_preimage);
+               nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &b_htlc_msgs.commitment_signed);
+               check_added_monitors!(nodes[0], 1);
+               let (as_raa, as_cs) = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
+               nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
+               let _htlc_updates = get_htlc_update_msgs!(&nodes[1], nodes[0].node.get_our_node_id());
+               check_added_monitors!(nodes[1], 1);
+               nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_cs);
+               let _bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
+               check_added_monitors!(nodes[1], 1);
+       }
+
+       // Once B has received the payment preimage, it includes the value of the HTLC in its
+       // "claimable if you were to close the channel" balance.
+       let mut a_expected_balances = vec![Balance::ClaimableOnChannelClose {
+                       claimable_amount_satoshis: 1_000_000 - // Channel funding value in satoshis
+                               4_000 - // The to-be-failed HTLC value in satoshis
+                               3_000 - // The claimed HTLC value in satoshis
+                               1_000 - // The push_msat value in satoshis
+                               3 - // The dust HTLC value in satoshis
+                               // The commitment transaction fee with two HTLC outputs:
+                               chan_feerate * (channel::COMMITMENT_TX_BASE_WEIGHT +
+                                                               if prev_commitment_tx { 1 } else { 2 } *
+                                                               channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }];
+       if !prev_commitment_tx {
+               a_expected_balances.push(Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 3_000,
+                       claimable_height: htlc_cltv_timeout,
+               });
+       }
+       assert_eq!(sorted_vec(a_expected_balances),
+               sorted_vec(nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+       assert_eq!(vec![Balance::ClaimableOnChannelClose {
+                       claimable_amount_satoshis: 1_000 + 3_000 + 4_000,
+               }],
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       // Broadcast the closing transaction (which has both pending HTLCs in it) and get B's
+       // broadcasted HTLC claim transaction with preimage.
+       let node_b_commitment_claimable = nodes[1].best_block_info().1 + BREAKDOWN_TIMEOUT as u32;
+       mine_transaction(&nodes[0], &remote_txn[0]);
+       mine_transaction(&nodes[1], &remote_txn[0]);
+
+       let b_broadcast_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+       assert_eq!(b_broadcast_txn.len(), if prev_commitment_tx { 4 } else { 5 });
+       if prev_commitment_tx {
+               check_spends!(b_broadcast_txn[3], b_broadcast_txn[2]);
+       } else {
+               assert_eq!(b_broadcast_txn[0], b_broadcast_txn[3]);
+               assert_eq!(b_broadcast_txn[1], b_broadcast_txn[4]);
+       }
+       // b_broadcast_txn[0] should spend the HTLC output of the commitment tx for 3_000 sats
+       check_spends!(b_broadcast_txn[0], remote_txn[0]);
+       check_spends!(b_broadcast_txn[1], remote_txn[0]);
+       assert_eq!(b_broadcast_txn[0].input.len(), 1);
+       assert_eq!(b_broadcast_txn[1].input.len(), 1);
+       assert_eq!(remote_txn[0].output[b_broadcast_txn[0].input[0].previous_output.vout as usize].value, 3_000);
+       assert_eq!(remote_txn[0].output[b_broadcast_txn[1].input[0].previous_output.vout as usize].value, 4_000);
+       check_spends!(b_broadcast_txn[2], funding_tx);
+
+       assert!(nodes[0].node.list_channels().is_empty());
+       check_closed_broadcast!(nodes[0], true);
+       check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
+       assert!(nodes[1].node.list_channels().is_empty());
+       check_closed_broadcast!(nodes[1], true);
+       check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
+       assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
+
+       // Once the commitment transaction confirms, we will wait until ANTI_REORG_DELAY until we
+       // generate any `SpendableOutputs` events. Thus, the same balances will still be listed
+       // available in `get_claimable_balances`. However, both will swap from `ClaimableOnClose` to
+       // other Balance variants, as close has already happened.
+       assert!(nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+
+       assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1_000_000 - 3_000 - 4_000 - 1_000 - 3 - chan_feerate *
+                               (channel::COMMITMENT_TX_BASE_WEIGHT + 2 * channel::COMMITMENT_TX_WEIGHT_PER_HTLC) / 1000,
+                       confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 3_000,
+                       claimable_height: htlc_cltv_timeout,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+       // The main non-HTLC balance is just awaiting confirmations, but the claimable height is the
+       // CSV delay, not ANTI_REORG_DELAY.
+       assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1_000,
+                       confirmation_height: node_b_commitment_claimable,
+               },
+               // Both HTLC balances are "contentious" as our counterparty could claim them if we wait too
+               // long.
+               Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 3_000,
+                       timeout_height: htlc_cltv_timeout,
+               }, Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+
+       connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
+       expect_payment_failed!(nodes[0], dust_payment_hash, true);
+       connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
+
+       // After ANTI_REORG_DELAY, A will consider its balance fully spendable and generate a
+       // `SpendableOutputs` event. However, B still has to wait for the CSV delay.
+       assert_eq!(sorted_vec(vec![Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 3_000,
+                       claimable_height: htlc_cltv_timeout,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+       assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1_000,
+                       confirmation_height: node_b_commitment_claimable,
+               }, Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 3_000,
+                       timeout_height: htlc_cltv_timeout,
+               }, Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+
+       let mut node_a_spendable = nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_a_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_a_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[0].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, remote_txn[0]);
+       }
+
+       assert!(nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events().is_empty());
+
+       // After broadcasting the HTLC claim transaction, node A will still consider the HTLC
+       // possibly-claimable up to ANTI_REORG_DELAY, at which point it will drop it.
+       mine_transaction(&nodes[0], &b_broadcast_txn[0]);
+       if !prev_commitment_tx {
+               expect_payment_sent!(nodes[0], payment_preimage);
+       }
+       assert_eq!(sorted_vec(vec![Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 3_000,
+                       claimable_height: htlc_cltv_timeout,
+               }, Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+       connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
+       assert_eq!(vec![Balance::MaybeClaimableHTLCAwaitingTimeout {
+                       claimable_amount_satoshis: 4_000,
+                       claimable_height: htlc_cltv_timeout,
+               }],
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       // When the HTLC timeout output is spendable in the next block, A should broadcast it
+       connect_blocks(&nodes[0], htlc_cltv_timeout - nodes[0].best_block_info().1 - 1);
+       let a_broadcast_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
+       assert_eq!(a_broadcast_txn.len(), 3);
+       check_spends!(a_broadcast_txn[0], funding_tx);
+       assert_eq!(a_broadcast_txn[1].input.len(), 1);
+       check_spends!(a_broadcast_txn[1], remote_txn[0]);
+       assert_eq!(a_broadcast_txn[2].input.len(), 1);
+       check_spends!(a_broadcast_txn[2], remote_txn[0]);
+       assert_ne!(a_broadcast_txn[1].input[0].previous_output.vout,
+                  a_broadcast_txn[2].input[0].previous_output.vout);
+       // a_broadcast_txn [1] and [2] should spend the HTLC outputs of the commitment tx
+       assert_eq!(remote_txn[0].output[a_broadcast_txn[1].input[0].previous_output.vout as usize].value, 3_000);
+       assert_eq!(remote_txn[0].output[a_broadcast_txn[2].input[0].previous_output.vout as usize].value, 4_000);
+
+       // Once the HTLC-Timeout transaction confirms, A will no longer consider the HTLC
+       // "MaybeClaimable", but instead move it to "AwaitingConfirmations".
+       mine_transaction(&nodes[0], &a_broadcast_txn[2]);
+       assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
+       assert_eq!(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 4_000,
+                       confirmation_height: nodes[0].best_block_info().1 + ANTI_REORG_DELAY - 1,
+               }],
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       // After ANTI_REORG_DELAY, A will generate a SpendableOutputs event and drop the claimable
+       // balance entry.
+       connect_blocks(&nodes[0], ANTI_REORG_DELAY - 1);
+       assert_eq!(Vec::<Balance>::new(),
+               nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       expect_payment_failed!(nodes[0], timeout_payment_hash, true);
+
+       let mut node_a_spendable = nodes[0].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_a_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_a_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[0].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, a_broadcast_txn[2]);
+       } else { panic!(); }
+
+       // Node B will no longer consider the HTLC "contentious" after the HTLC claim transaction
+       // confirms, and consider it simply "awaiting confirmations". Note that it has to wait for the
+       // standard revocable transaction CSV delay before receiving a `SpendableOutputs`.
+       let node_b_htlc_claimable = nodes[1].best_block_info().1 + BREAKDOWN_TIMEOUT as u32;
+       mine_transaction(&nodes[1], &b_broadcast_txn[0]);
+
+       assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 1_000,
+                       confirmation_height: node_b_commitment_claimable,
+               }, Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 3_000,
+                       confirmation_height: node_b_htlc_claimable,
+               }, Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+
+       // After reaching the commitment output CSV, we'll get a SpendableOutputs event for it and have
+       // only the HTLCs claimable on node B.
+       connect_blocks(&nodes[1], node_b_commitment_claimable - nodes[1].best_block_info().1);
+
+       let mut node_b_spendable = nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_b_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_b_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[1].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, remote_txn[0]);
+       }
+
+       assert_eq!(sorted_vec(vec![Balance::ClaimableAwaitingConfirmations {
+                       claimable_amount_satoshis: 3_000,
+                       confirmation_height: node_b_htlc_claimable,
+               }, Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }]),
+               sorted_vec(nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances()));
+
+       // After reaching the claimed HTLC output CSV, we'll get a SpendableOutptus event for it and
+       // have only one HTLC output left spendable.
+       connect_blocks(&nodes[1], node_b_htlc_claimable - nodes[1].best_block_info().1);
+
+       let mut node_b_spendable = nodes[1].chain_monitor.chain_monitor.get_and_clear_pending_events();
+       assert_eq!(node_b_spendable.len(), 1);
+       if let Event::SpendableOutputs { outputs } = node_b_spendable.pop().unwrap() {
+               assert_eq!(outputs.len(), 1);
+               let spend_tx = nodes[1].keys_manager.backing.spend_spendable_outputs(&[&outputs[0]], Vec::new(),
+                       Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script(), 253, &Secp256k1::new()).unwrap();
+               check_spends!(spend_tx, b_broadcast_txn[0]);
+       } else { panic!(); }
+
+       assert_eq!(vec![Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }],
+       nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+
+       // Finally, mine the HTLC timeout transaction that A broadcasted (even though B should be able
+       // to claim this HTLC with the preimage it knows!). It will remain listed as a claimable HTLC
+       // until ANTI_REORG_DELAY confirmations on the spend.
+       mine_transaction(&nodes[1], &a_broadcast_txn[2]);
+       assert_eq!(vec![Balance::ContentiousClaimable {
+                       claimable_amount_satoshis: 4_000,
+                       timeout_height: htlc_cltv_timeout,
+               }],
+       nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+       connect_blocks(&nodes[1], ANTI_REORG_DELAY - 1);
+       assert_eq!(Vec::<Balance>::new(),
+               nodes[1].chain_monitor.chain_monitor.monitors.read().unwrap().get(&funding_outpoint).unwrap().get_claimable_balances());
+}
+
+#[test]
+fn test_claim_value_force_close() {
+       do_test_claim_value_force_close(true);
+       do_test_claim_value_force_close(false);
 }
index 5b49f43b1188013c5f44822e59622409590aefb3..122dd7737b996507e2d8fb5135af443172fb65f8 100644 (file)
@@ -745,35 +745,6 @@ pub struct CommitmentUpdate {
        pub commitment_signed: CommitmentSigned,
 }
 
-/// The information we received from a peer along the route of a payment we originated. This is
-/// returned by ChannelMessageHandler::handle_update_fail_htlc to be passed into
-/// RoutingMessageHandler::handle_htlc_fail_channel_update to update our network map.
-#[derive(Clone, Debug, PartialEq)]
-pub enum HTLCFailChannelUpdate {
-       /// We received an error which included a full ChannelUpdate message.
-       ChannelUpdateMessage {
-               /// The unwrapped message we received
-               msg: ChannelUpdate,
-       },
-       /// We received an error which indicated only that a channel has been closed
-       ChannelClosed {
-               /// The short_channel_id which has now closed.
-               short_channel_id: u64,
-               /// when this true, this channel should be permanently removed from the
-               /// consideration. Otherwise, this channel can be restored as new channel_update is received
-               is_permanent: bool,
-       },
-       /// We received an error which indicated only that a node has failed
-       NodeFailure {
-               /// The node_id that has failed.
-               node_id: PublicKey,
-               /// when this true, node should be permanently removed from the
-               /// consideration. Otherwise, the channels connected to this node can be
-               /// restored as new channel_update is received
-               is_permanent: bool,
-       }
-}
-
 /// Messages could have optional fields to use with extended features
 /// As we wish to serialize these differently from Option<T>s (Options get a tag byte, but
 /// OptionalFeild simply gets Present if there are enough bytes to read into it), we have a
@@ -868,8 +839,6 @@ pub trait RoutingMessageHandler : MessageSendEventsProvider {
        /// Handle an incoming channel_update message, returning true if it should be forwarded on,
        /// false or returning an Err otherwise.
        fn handle_channel_update(&self, msg: &ChannelUpdate) -> Result<bool, LightningError>;
-       /// Handle some updates to the route graph that we learned due to an outbound failed payment.
-       fn handle_htlc_fail_channel_update(&self, update: &HTLCFailChannelUpdate);
        /// Gets a subset of the channel announcements and updates required to dump our routing table
        /// to a remote node, starting at the short_channel_id indicated by starting_point and
        /// including the batch_amount entries immediately higher in numerical value than starting_point.
@@ -1053,10 +1022,7 @@ impl Readable for OptionalField<u64> {
 }
 
 
-impl_writeable_len_match!(AcceptChannel, {
-               {AcceptChannel{ shutdown_scriptpubkey: OptionalField::Present(ref script), .. }, 270 + 2 + script.len()},
-               {_, 270}
-       }, {
+impl_writeable_msg!(AcceptChannel, {
        temporary_channel_id,
        dust_limit_satoshis,
        max_htlc_value_in_flight_msat,
@@ -1072,18 +1038,17 @@ impl_writeable_len_match!(AcceptChannel, {
        htlc_basepoint,
        first_per_commitment_point,
        shutdown_scriptpubkey
-});
+}, {});
 
-impl_writeable!(AnnouncementSignatures, 32+8+64*2, {
+impl_writeable_msg!(AnnouncementSignatures, {
        channel_id,
        short_channel_id,
        node_signature,
        bitcoin_signature
-});
+}, {});
 
 impl Writeable for ChannelReestablish {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(if let OptionalField::Present(..) = self.data_loss_protect { 32+2*8+33+32 } else { 32+2*8 });
                self.channel_id.write(w)?;
                self.next_local_commitment_number.write(w)?;
                self.next_remote_commitment_number.write(w)?;
@@ -1119,69 +1084,44 @@ impl Readable for ChannelReestablish{
        }
 }
 
-impl Writeable for ClosingSigned {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 8 + 64 + if self.fee_range.is_some() { 1+1+ 2*8 } else { 0 });
-               self.channel_id.write(w)?;
-               self.fee_satoshis.write(w)?;
-               self.signature.write(w)?;
-               encode_tlv_stream!(w, {
-                       (1, self.fee_range, option),
-               });
-               Ok(())
-       }
-}
+impl_writeable_msg!(ClosingSigned,
+       { channel_id, fee_satoshis, signature },
+       { (1, fee_range, option) }
+);
 
-impl Readable for ClosingSigned {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let channel_id = Readable::read(r)?;
-               let fee_satoshis = Readable::read(r)?;
-               let signature = Readable::read(r)?;
-               let mut fee_range = None;
-               decode_tlv_stream!(r, {
-                       (1, fee_range, option),
-               });
-               Ok(Self { channel_id, fee_satoshis, signature, fee_range })
-       }
-}
-
-impl_writeable!(ClosingSignedFeeRange, 2*8, {
+impl_writeable!(ClosingSignedFeeRange, {
        min_fee_satoshis,
        max_fee_satoshis
 });
 
-impl_writeable_len_match!(CommitmentSigned, {
-               { CommitmentSigned { ref htlc_signatures, .. }, 32+64+2+htlc_signatures.len()*64 }
-       }, {
+impl_writeable_msg!(CommitmentSigned, {
        channel_id,
        signature,
        htlc_signatures
-});
+}, {});
 
-impl_writeable_len_match!(DecodedOnionErrorPacket, {
-               { DecodedOnionErrorPacket { ref failuremsg, ref pad, .. }, 32 + 4 + failuremsg.len() + pad.len() }
-       }, {
+impl_writeable!(DecodedOnionErrorPacket, {
        hmac,
        failuremsg,
        pad
 });
 
-impl_writeable!(FundingCreated, 32+32+2+64, {
+impl_writeable_msg!(FundingCreated, {
        temporary_channel_id,
        funding_txid,
        funding_output_index,
        signature
-});
+}, {});
 
-impl_writeable!(FundingSigned, 32+64, {
+impl_writeable_msg!(FundingSigned, {
        channel_id,
        signature
-});
+}, {});
 
-impl_writeable!(FundingLocked, 32+33, {
+impl_writeable_msg!(FundingLocked, {
        channel_id,
-       next_per_commitment_point
-});
+       next_per_commitment_point,
+}, {});
 
 impl Writeable for Init {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
@@ -1202,10 +1142,7 @@ impl Readable for Init {
        }
 }
 
-impl_writeable_len_match!(OpenChannel, {
-               { OpenChannel { shutdown_scriptpubkey: OptionalField::Present(ref script), .. }, 319 + 2 + script.len() },
-               { _, 319 }
-       }, {
+impl_writeable_msg!(OpenChannel, {
        chain_hash,
        temporary_channel_id,
        funding_satoshis,
@@ -1225,56 +1162,55 @@ impl_writeable_len_match!(OpenChannel, {
        first_per_commitment_point,
        channel_flags,
        shutdown_scriptpubkey
-});
+}, {});
 
-impl_writeable!(RevokeAndACK, 32+32+33, {
+impl_writeable_msg!(RevokeAndACK, {
        channel_id,
        per_commitment_secret,
        next_per_commitment_point
-});
+}, {});
 
-impl_writeable_len_match!(Shutdown, {
-               { Shutdown { ref scriptpubkey, .. }, 32 + 2 + scriptpubkey.len() }
-       }, {
+impl_writeable_msg!(Shutdown, {
        channel_id,
        scriptpubkey
-});
+}, {});
 
-impl_writeable_len_match!(UpdateFailHTLC, {
-               { UpdateFailHTLC { ref reason, .. }, 32 + 10 + reason.data.len() }
-       }, {
+impl_writeable_msg!(UpdateFailHTLC, {
        channel_id,
        htlc_id,
        reason
-});
+}, {});
 
-impl_writeable!(UpdateFailMalformedHTLC, 32+8+32+2, {
+impl_writeable_msg!(UpdateFailMalformedHTLC, {
        channel_id,
        htlc_id,
        sha256_of_onion,
        failure_code
-});
+}, {});
 
-impl_writeable!(UpdateFee, 32+4, {
+impl_writeable_msg!(UpdateFee, {
        channel_id,
        feerate_per_kw
-});
+}, {});
 
-impl_writeable!(UpdateFulfillHTLC, 32+8+32, {
+impl_writeable_msg!(UpdateFulfillHTLC, {
        channel_id,
        htlc_id,
        payment_preimage
-});
+}, {});
 
-impl_writeable_len_match!(OnionErrorPacket, {
-               { OnionErrorPacket { ref data, .. }, 2 + data.len() }
-       }, {
+// Note that this is written as a part of ChannelManager objects, and thus cannot change its
+// serialization format in a way which assumes we know the total serialized length/message end
+// position.
+impl_writeable!(OnionErrorPacket, {
        data
 });
 
+// Note that this is written as a part of ChannelManager objects, and thus cannot change its
+// serialization format in a way which assumes we know the total serialized length/message end
+// position.
 impl Writeable for OnionPacket {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(1 + 33 + 20*65 + 32);
                self.version.write(w)?;
                match self.public_key {
                        Ok(pubkey) => pubkey.write(w)?,
@@ -1301,18 +1237,17 @@ impl Readable for OnionPacket {
        }
 }
 
-impl_writeable!(UpdateAddHTLC, 32+8+8+32+4+1366, {
+impl_writeable_msg!(UpdateAddHTLC, {
        channel_id,
        htlc_id,
        amount_msat,
        payment_hash,
        cltv_expiry,
        onion_routing_packet
-});
+}, {});
 
 impl Writeable for FinalOnionHopData {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 8 - (self.total_msat.leading_zeros()/8) as usize);
                self.payment_secret.0.write(w)?;
                HighZeroBytesDroppedVarInt(self.total_msat).write(w)
        }
@@ -1328,7 +1263,6 @@ impl Readable for FinalOnionHopData {
 
 impl Writeable for OnionHopData {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(33);
                // Note that this should never be reachable if Rust-Lightning generated the message, as we
                // check values are sane long before we get here, though its possible in the future
                // user-generated messages may hit this.
@@ -1429,7 +1363,6 @@ impl Readable for OnionHopData {
 
 impl Writeable for Ping {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(self.byteslen as usize + 4);
                self.ponglen.write(w)?;
                vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
                Ok(())
@@ -1451,7 +1384,6 @@ impl Readable for Ping {
 
 impl Writeable for Pong {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(self.byteslen as usize + 2);
                vec![0u8; self.byteslen as usize].write(w)?; // size-unchecked write
                Ok(())
        }
@@ -1471,7 +1403,6 @@ impl Readable for Pong {
 
 impl Writeable for UnsignedChannelAnnouncement {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(2 + 32 + 8 + 4*33 + self.features.byte_count() + self.excess_data.len());
                self.features.write(w)?;
                self.chain_hash.write(w)?;
                self.short_channel_id.write(w)?;
@@ -1499,10 +1430,7 @@ impl Readable for UnsignedChannelAnnouncement {
        }
 }
 
-impl_writeable_len_match!(ChannelAnnouncement, {
-               { ChannelAnnouncement { contents: UnsignedChannelAnnouncement {ref features, ref excess_data, ..}, .. },
-                       2 + 32 + 8 + 4*33 + features.byte_count() + excess_data.len() + 4*64 }
-       }, {
+impl_writeable!(ChannelAnnouncement, {
        node_signature_1,
        node_signature_2,
        bitcoin_signature_1,
@@ -1512,13 +1440,10 @@ impl_writeable_len_match!(ChannelAnnouncement, {
 
 impl Writeable for UnsignedChannelUpdate {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               let mut size = 64 + self.excess_data.len();
                let mut message_flags: u8 = 0;
                if let OptionalField::Present(_) = self.htlc_maximum_msat {
-                       size += 8;
                        message_flags = 1;
                }
-               w.size_hint(size);
                self.chain_hash.write(w)?;
                self.short_channel_id.write(w)?;
                self.timestamp.write(w)?;
@@ -1557,17 +1482,13 @@ impl Readable for UnsignedChannelUpdate {
        }
 }
 
-impl_writeable_len_match!(ChannelUpdate, {
-               { ChannelUpdate { contents: UnsignedChannelUpdate {ref excess_data, ref htlc_maximum_msat, ..}, .. },
-                       64 + 64 + excess_data.len() + if let OptionalField::Present(_) = htlc_maximum_msat { 8 } else { 0 } }
-       }, {
+impl_writeable!(ChannelUpdate, {
        signature,
        contents
 });
 
 impl Writeable for ErrorMessage {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 2 + self.data.len());
                self.channel_id.write(w)?;
                (self.data.len() as u16).write(w)?;
                w.write_all(self.data.as_bytes())?;
@@ -1594,7 +1515,6 @@ impl Readable for ErrorMessage {
 
 impl Writeable for UnsignedNodeAnnouncement {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(76 + self.features.byte_count() + self.addresses.len()*38 + self.excess_address_data.len() + self.excess_data.len());
                self.features.write(w)?;
                self.timestamp.write(w)?;
                self.node_id.write(w)?;
@@ -1677,10 +1597,7 @@ impl Readable for UnsignedNodeAnnouncement {
        }
 }
 
-impl_writeable_len_match!(NodeAnnouncement, <=, {
-               { NodeAnnouncement { contents: UnsignedNodeAnnouncement { ref features, ref addresses, ref excess_address_data, ref excess_data, ..}, .. },
-                       64 + 76 + features.byte_count() + addresses.len()*(NetAddress::MAX_LEN as usize + 1) + excess_address_data.len() + excess_data.len() }
-       }, {
+impl_writeable!(NodeAnnouncement, {
        signature,
        contents
 });
@@ -1724,7 +1641,6 @@ impl Writeable for QueryShortChannelIds {
                // Calculated from 1-byte encoding_type plus 8-bytes per short_channel_id
                let encoding_len: u16 = 1 + self.short_channel_ids.len() as u16 * 8;
 
-               w.size_hint(32 + 2 + encoding_len as usize);
                self.chain_hash.write(w)?;
                encoding_len.write(w)?;
 
@@ -1739,25 +1655,10 @@ impl Writeable for QueryShortChannelIds {
        }
 }
 
-impl Readable for ReplyShortChannelIdsEnd {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let full_information: bool = Readable::read(r)?;
-               Ok(ReplyShortChannelIdsEnd {
-                       chain_hash,
-                       full_information,
-               })
-       }
-}
-
-impl Writeable for ReplyShortChannelIdsEnd {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 1);
-               self.chain_hash.write(w)?;
-               self.full_information.write(w)?;
-               Ok(())
-       }
-}
+impl_writeable_msg!(ReplyShortChannelIdsEnd, {
+       chain_hash,
+       full_information,
+}, {});
 
 impl QueryChannelRange {
        /**
@@ -1772,28 +1673,11 @@ impl QueryChannelRange {
        }
 }
 
-impl Readable for QueryChannelRange {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let first_blocknum: u32 = Readable::read(r)?;
-               let number_of_blocks: u32 = Readable::read(r)?;
-               Ok(QueryChannelRange {
-                       chain_hash,
-                       first_blocknum,
-                       number_of_blocks
-               })
-       }
-}
-
-impl Writeable for QueryChannelRange {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 4 + 4);
-               self.chain_hash.write(w)?;
-               self.first_blocknum.write(w)?;
-               self.number_of_blocks.write(w)?;
-               Ok(())
-       }
-}
+impl_writeable_msg!(QueryChannelRange, {
+       chain_hash,
+       first_blocknum,
+       number_of_blocks
+}, {});
 
 impl Readable for ReplyChannelRange {
        fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
@@ -1838,7 +1722,6 @@ impl Readable for ReplyChannelRange {
 impl Writeable for ReplyChannelRange {
        fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
                let encoding_len: u16 = 1 + self.short_channel_ids.len() as u16 * 8;
-               w.size_hint(32 + 4 + 4 + 1 + 2 + encoding_len as usize);
                self.chain_hash.write(w)?;
                self.first_blocknum.write(w)?;
                self.number_of_blocks.write(w)?;
@@ -1854,29 +1737,11 @@ impl Writeable for ReplyChannelRange {
        }
 }
 
-impl Readable for GossipTimestampFilter {
-       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
-               let chain_hash: BlockHash = Readable::read(r)?;
-               let first_timestamp: u32 = Readable::read(r)?;
-               let timestamp_range: u32 = Readable::read(r)?;
-               Ok(GossipTimestampFilter {
-                       chain_hash,
-                       first_timestamp,
-                       timestamp_range,
-               })
-       }
-}
-
-impl Writeable for GossipTimestampFilter {
-       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
-               w.size_hint(32 + 4 + 4);
-               self.chain_hash.write(w)?;
-               self.first_timestamp.write(w)?;
-               self.timestamp_range.write(w)?;
-               Ok(())
-       }
-}
-
+impl_writeable_msg!(GossipTimestampFilter, {
+       chain_hash,
+       first_timestamp,
+       timestamp_range,
+}, {});
 
 #[cfg(test)]
 mod tests {
index 8530d3d6610c181bfac5c1b1cab0f493090bb510..0e2d081c83bf862c7a3a80165c9f78b8e87ba1ec 100644 (file)
@@ -16,9 +16,10 @@ use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
 use ln::channelmanager::{HTLCForwardInfo, CLTV_FAR_FAR_AWAY};
 use ln::onion_utils;
 use routing::router::{Route, get_route};
+use routing::network_graph::NetworkUpdate;
 use ln::features::{InitFeatures, InvoiceFeatures};
 use ln::msgs;
-use ln::msgs::{ChannelMessageHandler, ChannelUpdate, HTLCFailChannelUpdate, OptionalField};
+use ln::msgs::{ChannelMessageHandler, ChannelUpdate, OptionalField};
 use util::test_utils;
 use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
 use util::ser::{Writeable, Writer};
@@ -39,7 +40,7 @@ use core::default::Default;
 
 use ln::functional_test_utils::*;
 
-fn run_onion_failure_test<F1,F2>(_name: &str, test_case: u8, nodes: &Vec<Node>, route: &Route, payment_hash: &PaymentHash, payment_secret: &PaymentSecret, callback_msg: F1, callback_node: F2, expected_retryable: bool, expected_error_code: Option<u16>, expected_channel_update: Option<HTLCFailChannelUpdate>)
+fn run_onion_failure_test<F1,F2>(_name: &str, test_case: u8, nodes: &Vec<Node>, route: &Route, payment_hash: &PaymentHash, payment_secret: &PaymentSecret, callback_msg: F1, callback_node: F2, expected_retryable: bool, expected_error_code: Option<u16>, expected_channel_update: Option<NetworkUpdate>)
        where F1: for <'a> FnMut(&'a mut msgs::UpdateAddHTLC),
                                F2: FnMut(),
 {
@@ -53,7 +54,7 @@ fn run_onion_failure_test<F1,F2>(_name: &str, test_case: u8, nodes: &Vec<Node>,
 // 3: final node fails backward (but tamper onion payloads from node0)
 // 100: trigger error in the intermediate node and tamper returning fail_htlc
 // 200: trigger error in the final node and tamper returning fail_htlc
-fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case: u8, nodes: &Vec<Node>, route: &Route, payment_hash: &PaymentHash, payment_secret: &PaymentSecret, mut callback_msg: F1, mut callback_fail: F2, mut callback_node: F3, expected_retryable: bool, expected_error_code: Option<u16>, expected_channel_update: Option<HTLCFailChannelUpdate>)
+fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case: u8, nodes: &Vec<Node>, route: &Route, payment_hash: &PaymentHash, payment_secret: &PaymentSecret, mut callback_msg: F1, mut callback_fail: F2, mut callback_node: F3, expected_retryable: bool, expected_error_code: Option<u16>, expected_channel_update: Option<NetworkUpdate>)
        where F1: for <'a> FnMut(&'a mut msgs::UpdateAddHTLC),
                                F2: for <'a> FnMut(&'a mut msgs::UpdateFailHTLC),
                                F3: FnMut(),
@@ -162,34 +163,28 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
 
        let events = nodes[0].node.get_and_clear_pending_events();
        assert_eq!(events.len(), 1);
-       if let &Event::PaymentFailed { payment_hash:_, ref rejected_by_dest, ref error_code, error_data: _ } = &events[0] {
+       if let &Event::PaymentPathFailed { payment_hash:_, ref rejected_by_dest, ref network_update, ref error_code, error_data: _, ref all_paths_failed, path: _ } = &events[0] {
                assert_eq!(*rejected_by_dest, !expected_retryable);
+               assert_eq!(*all_paths_failed, true);
                assert_eq!(*error_code, expected_error_code);
-       } else {
-               panic!("Uexpected event");
-       }
-
-       let events = nodes[0].node.get_and_clear_pending_msg_events();
-       if expected_channel_update.is_some() {
-               assert_eq!(events.len(), 1);
-               match events[0] {
-                       MessageSendEvent::PaymentFailureNetworkUpdate { ref update } => {
-                               match update {
-                                       &HTLCFailChannelUpdate::ChannelUpdateMessage { .. } => {
-                                               if let HTLCFailChannelUpdate::ChannelUpdateMessage { .. } = expected_channel_update.unwrap() {} else {
+               if expected_channel_update.is_some() {
+                       match network_update {
+                               Some(update) => match update {
+                                       &NetworkUpdate::ChannelUpdateMessage { .. } => {
+                                               if let NetworkUpdate::ChannelUpdateMessage { .. } = expected_channel_update.unwrap() {} else {
                                                        panic!("channel_update not found!");
                                                }
                                        },
-                                       &HTLCFailChannelUpdate::ChannelClosed { ref short_channel_id, ref is_permanent } => {
-                                               if let HTLCFailChannelUpdate::ChannelClosed { short_channel_id: ref expected_short_channel_id, is_permanent: ref expected_is_permanent } = expected_channel_update.unwrap() {
+                                       &NetworkUpdate::ChannelClosed { ref short_channel_id, ref is_permanent } => {
+                                               if let NetworkUpdate::ChannelClosed { short_channel_id: ref expected_short_channel_id, is_permanent: ref expected_is_permanent } = expected_channel_update.unwrap() {
                                                        assert!(*short_channel_id == *expected_short_channel_id);
                                                        assert!(*is_permanent == *expected_is_permanent);
                                                } else {
                                                        panic!("Unexpected message event");
                                                }
                                        },
-                                       &HTLCFailChannelUpdate::NodeFailure { ref node_id, ref is_permanent } => {
-                                               if let HTLCFailChannelUpdate::NodeFailure { node_id: ref expected_node_id, is_permanent: ref expected_is_permanent } = expected_channel_update.unwrap() {
+                                       &NetworkUpdate::NodeFailure { ref node_id, ref is_permanent } => {
+                                               if let NetworkUpdate::NodeFailure { node_id: ref expected_node_id, is_permanent: ref expected_is_permanent } = expected_channel_update.unwrap() {
                                                        assert!(*node_id == *expected_node_id);
                                                        assert!(*is_permanent == *expected_is_permanent);
                                                } else {
@@ -197,11 +192,13 @@ fn run_onion_failure_test_with_fail_intercept<F1,F2,F3>(_name: &str, test_case:
                                                }
                                        },
                                }
-                       },
-                       _ => panic!("Unexpected message event"),
+                               None => panic!("Expected channel update"),
+                       }
+               } else {
+                       assert!(network_update.is_none());
                }
        } else {
-               assert_eq!(events.len(), 0);
+               panic!("Unexpected event");
        }
 }
 
@@ -263,7 +260,7 @@ fn test_fee_failures() {
        let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
        let channels = [create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()), create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known())];
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40_000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40_000, TEST_FINAL_CLTV, &logger).unwrap();
 
        // positive case
        let (payment_preimage_success, payment_hash_success, payment_secret_success) = get_payment_preimage_hash!(nodes[2]);
@@ -275,7 +272,7 @@ fn test_fee_failures() {
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
        run_onion_failure_test("fee_insufficient", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                msg.amount_msat -= 1;
-       }, || {}, true, Some(UPDATE|12), Some(msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
+       }, || {}, true, Some(UPDATE|12), Some(NetworkUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
 
        // In an earlier version, we spuriously failed to forward payments if the expected feerate
        // changed between the channel open and the payment.
@@ -318,7 +315,7 @@ fn test_onion_failure() {
        let channels = [create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()), create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known())];
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 40000, TEST_FINAL_CLTV, &logger).unwrap();
        // positve case
        send_payment(&nodes[0], &vec!(&nodes[1], &nodes[2])[..], 40000);
 
@@ -336,7 +333,7 @@ fn test_onion_failure() {
                // describing a length-1 TLV payload, which is obviously bogus.
                new_payloads[0].data[0] = 1;
                msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
-       }, ||{}, true, Some(PERM|22), Some(msgs::HTLCFailChannelUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));//XXX incremented channels idx here
+       }, ||{}, true, Some(PERM|22), Some(NetworkUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
 
        // final node failure
        run_onion_failure_test("invalid_realm", 3, &nodes, &route, &payment_hash, &payment_secret, |msg| {
@@ -352,7 +349,7 @@ fn test_onion_failure() {
                // length-1 TLV payload, which is obviously bogus.
                new_payloads[1].data[0] = 1;
                msg.onion_routing_packet = onion_utils::construct_onion_packet_bogus_hopdata(new_payloads, onion_keys, [0; 32], &payment_hash);
-       }, ||{}, false, Some(PERM|22), Some(msgs::HTLCFailChannelUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
+       }, ||{}, false, Some(PERM|22), Some(NetworkUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
 
        // the following three with run_onion_failure_test_with_fail_intercept() test only the origin node
        // receiving simulated fail messages
@@ -365,7 +362,7 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], NODE|2, &[0;0]);
-       }, ||{}, true, Some(NODE|2), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: false}));
+       }, ||{}, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: false}));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("temporary_node_failure", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -375,7 +372,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[1].shared_secret[..], NODE|2, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, true, Some(NODE|2), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: false}));
+       }, true, Some(NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: false}));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        // intermediate node failure
@@ -385,7 +382,7 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], PERM|NODE|2, &[0;0]);
-       }, ||{}, true, Some(PERM|NODE|2), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}));
+       }, ||{}, true, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("permanent_node_failure", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -394,7 +391,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[1].shared_secret[..], PERM|NODE|2, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, false, Some(PERM|NODE|2), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}));
+       }, false, Some(PERM|NODE|2), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        // intermediate node failure
@@ -406,7 +403,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], PERM|NODE|3, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, true, Some(PERM|NODE|3), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}));
+       }, true, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][0].pubkey, is_permanent: true}));
 
        // final node failure
        run_onion_failure_test_with_fail_intercept("required_node_feature_missing", 200, &nodes, &route, &payment_hash, &payment_secret, |_msg| {}, |msg| {
@@ -415,7 +412,7 @@ fn test_onion_failure() {
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[1].shared_secret[..], PERM|NODE|3, &[0;0]);
        }, ||{
                nodes[2].node.fail_htlc_backwards(&payment_hash);
-       }, false, Some(PERM|NODE|3), Some(msgs::HTLCFailChannelUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}));
+       }, false, Some(PERM|NODE|3), Some(NetworkUpdate::NodeFailure{node_id: route.paths[0][1].pubkey, is_permanent: true}));
        let (_, payment_hash, payment_secret) = get_payment_preimage_hash!(nodes[2]);
 
        run_onion_failure_test("invalid_onion_version", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| { msg.onion_routing_packet.version = 1; }, ||{}, true,
@@ -433,7 +430,7 @@ fn test_onion_failure() {
                let session_priv = SecretKey::from_slice(&[3; 32]).unwrap();
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], UPDATE|7, &ChannelUpdate::dummy().encode_with_len()[..]);
-       }, ||{}, true, Some(UPDATE|7), Some(msgs::HTLCFailChannelUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
+       }, ||{}, true, Some(UPDATE|7), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
 
        run_onion_failure_test_with_fail_intercept("permanent_channel_failure", 100, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                msg.amount_msat -= 1;
@@ -442,7 +439,7 @@ fn test_onion_failure() {
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], PERM|8, &[0;0]);
                // short_channel_id from the processing node
-       }, ||{}, true, Some(PERM|8), Some(msgs::HTLCFailChannelUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
+       }, ||{}, true, Some(PERM|8), Some(NetworkUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
 
        run_onion_failure_test_with_fail_intercept("required_channel_feature_missing", 100, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                msg.amount_msat -= 1;
@@ -451,18 +448,18 @@ fn test_onion_failure() {
                let onion_keys = onion_utils::construct_onion_keys(&Secp256k1::new(), &route.paths[0], &session_priv).unwrap();
                msg.reason = onion_utils::build_first_hop_failure_packet(&onion_keys[0].shared_secret[..], PERM|9, &[0;0]);
                // short_channel_id from the processing node
-       }, ||{}, true, Some(PERM|9), Some(msgs::HTLCFailChannelUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
+       }, ||{}, true, Some(PERM|9), Some(NetworkUpdate::ChannelClosed{short_channel_id: channels[1].0.contents.short_channel_id, is_permanent: true}));
 
        let mut bogus_route = route.clone();
        bogus_route.paths[0][1].short_channel_id -= 1;
        run_onion_failure_test("unknown_next_peer", 0, &nodes, &bogus_route, &payment_hash, &payment_secret, |_| {}, ||{}, true, Some(PERM|10),
-         Some(msgs::HTLCFailChannelUpdate::ChannelClosed{short_channel_id: bogus_route.paths[0][1].short_channel_id, is_permanent:true}));
+         Some(NetworkUpdate::ChannelClosed{short_channel_id: bogus_route.paths[0][1].short_channel_id, is_permanent:true}));
 
        let amt_to_forward = nodes[1].node.channel_state.lock().unwrap().by_id.get(&channels[1].2).unwrap().get_counterparty_htlc_minimum_msat() - 1;
        let mut bogus_route = route.clone();
        let route_len = bogus_route.paths[0].len();
        bogus_route.paths[0][route_len-1].fee_msat = amt_to_forward;
-       run_onion_failure_test("amount_below_minimum", 0, &nodes, &bogus_route, &payment_hash, &payment_secret, |_| {}, ||{}, true, Some(UPDATE|11), Some(msgs::HTLCFailChannelUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
+       run_onion_failure_test("amount_below_minimum", 0, &nodes, &bogus_route, &payment_hash, &payment_secret, |_| {}, ||{}, true, Some(UPDATE|11), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
 
        // Test a positive test-case with one extra msat, meeting the minimum.
        bogus_route.paths[0][route_len-1].fee_msat = amt_to_forward + 1;
@@ -473,19 +470,19 @@ fn test_onion_failure() {
        //invalid channel_update cases.
        run_onion_failure_test("fee_insufficient", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                msg.amount_msat -= 1;
-       }, || {}, true, Some(UPDATE|12), Some(msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
+       }, || {}, true, Some(UPDATE|12), Some(NetworkUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
 
        run_onion_failure_test("incorrect_cltv_expiry", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                // need to violate: cltv_expiry - cltv_expiry_delta >= outgoing_cltv_value
                msg.cltv_expiry -= 1;
-       }, || {}, true, Some(UPDATE|13), Some(msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
+       }, || {}, true, Some(UPDATE|13), Some(NetworkUpdate::ChannelClosed { short_channel_id: channels[0].0.contents.short_channel_id, is_permanent: true}));
 
        run_onion_failure_test("expiry_too_soon", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| {
                let height = msg.cltv_expiry - CLTV_CLAIM_BUFFER - LATENCY_GRACE_PERIOD_BLOCKS + 1;
                connect_blocks(&nodes[0], height - nodes[0].best_block_info().1);
                connect_blocks(&nodes[1], height - nodes[1].best_block_info().1);
                connect_blocks(&nodes[2], height - nodes[2].best_block_info().1);
-       }, ||{}, true, Some(UPDATE|14), Some(msgs::HTLCFailChannelUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
+       }, ||{}, true, Some(UPDATE|14), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
 
        run_onion_failure_test("unknown_payment_hash", 2, &nodes, &route, &payment_hash, &payment_secret, |_| {}, || {
                nodes[2].node.fail_htlc_backwards(&payment_hash);
@@ -528,7 +525,7 @@ fn test_onion_failure() {
                // disconnect event to the channel between nodes[1] ~ nodes[2]
                nodes[1].node.peer_disconnected(&nodes[2].node.get_our_node_id(), false);
                nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
-       }, true, Some(UPDATE|20), Some(msgs::HTLCFailChannelUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
+       }, true, Some(UPDATE|20), Some(NetworkUpdate::ChannelUpdateMessage{msg: ChannelUpdate::dummy()}));
        reconnect_nodes(&nodes[1], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 
        run_onion_failure_test("expiry_too_far", 0, &nodes, &route, &payment_hash, &payment_secret, |msg| {
index 4886168dd16cc277adfd536fcedbca83ba03ac1a..ee3ed96b5ef0efdcaaeda40a32f2e71653948ac0 100644 (file)
@@ -10,6 +10,7 @@
 use ln::{PaymentHash, PaymentPreimage, PaymentSecret};
 use ln::channelmanager::HTLCSource;
 use ln::msgs;
+use routing::network_graph::NetworkUpdate;
 use routing::router::RouteHop;
 use util::chacha20::ChaCha20;
 use util::errors::{self, APIError};
@@ -330,8 +331,8 @@ pub(super) fn build_first_hop_failure_packet(shared_secret: &[u8], failure_type:
 /// OutboundRoute).
 /// Returns update, a boolean indicating that the payment itself failed, and the error code.
 #[inline]
-pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &Secp256k1<T>, logger: &L, htlc_source: &HTLCSource, mut packet_decrypted: Vec<u8>) -> (Option<msgs::HTLCFailChannelUpdate>, bool, Option<u16>, Option<Vec<u8>>) where L::Target: Logger {
-       if let &HTLCSource::OutboundRoute { ref path, ref session_priv, ref first_hop_htlc_msat } = htlc_source {
+pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &Secp256k1<T>, logger: &L, htlc_source: &HTLCSource, mut packet_decrypted: Vec<u8>) -> (Option<NetworkUpdate>, bool, Option<u16>, Option<Vec<u8>>) where L::Target: Logger {
+       if let &HTLCSource::OutboundRoute { ref path, ref session_priv, ref first_hop_htlc_msat, .. } = htlc_source {
                let mut res = None;
                let mut htlc_msat = *first_hop_htlc_msat;
                let mut error_code_ret = None;
@@ -382,13 +383,13 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                } && is_from_final_node) // PERM bit observed below even this error is from the intermediate nodes
                                                || error_code == 21; // Special case error 21 as the Route object is bogus, TODO: Maybe fail the node if the CLTV was reasonable?
 
-                                               let mut fail_channel_update = None;
+                                               let mut network_update = None;
 
                                                if error_code & NODE == NODE {
-                                                       fail_channel_update = Some(msgs::HTLCFailChannelUpdate::NodeFailure { node_id: route_hop.pubkey, is_permanent: error_code & PERM == PERM });
+                                                       network_update = Some(NetworkUpdate::NodeFailure { node_id: route_hop.pubkey, is_permanent: error_code & PERM == PERM });
                                                }
                                                else if error_code & PERM == PERM {
-                                                       fail_channel_update = if payment_failed {None} else {Some(msgs::HTLCFailChannelUpdate::ChannelClosed {
+                                                       network_update = if payment_failed { None } else { Some(NetworkUpdate::ChannelClosed {
                                                                short_channel_id: path[next_route_hop_ix - if next_route_hop_ix == path.len() { 1 } else { 0 }].short_channel_id,
                                                                is_permanent: true,
                                                        })};
@@ -412,25 +413,25 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                                                        20 => chan_update.contents.flags & 2 == 0,
                                                                                        _ => false, // unknown error code; take channel_update as valid
                                                                                };
-                                                                               fail_channel_update = if is_chan_update_invalid {
+                                                                               network_update = if is_chan_update_invalid {
                                                                                        // This probably indicates the node which forwarded
                                                                                        // to the node in question corrupted something.
-                                                                                       Some(msgs::HTLCFailChannelUpdate::ChannelClosed {
+                                                                                       Some(NetworkUpdate::ChannelClosed {
                                                                                                short_channel_id: route_hop.short_channel_id,
                                                                                                is_permanent: true,
                                                                                        })
                                                                                } else {
-                                                                                       Some(msgs::HTLCFailChannelUpdate::ChannelUpdateMessage {
+                                                                                       Some(NetworkUpdate::ChannelUpdateMessage {
                                                                                                msg: chan_update,
                                                                                        })
                                                                                };
                                                                        }
                                                                }
                                                        }
-                                                       if fail_channel_update.is_none() {
+                                                       if network_update.is_none() {
                                                                // They provided an UPDATE which was obviously bogus, not worth
                                                                // trying to relay through them anymore.
-                                                               fail_channel_update = Some(msgs::HTLCFailChannelUpdate::NodeFailure {
+                                                               network_update = Some(NetworkUpdate::NodeFailure {
                                                                        node_id: route_hop.pubkey,
                                                                        is_permanent: true,
                                                                });
@@ -439,7 +440,7 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                        // We can't understand their error messages and they failed to
                                                        // forward...they probably can't understand our forwards so its
                                                        // really not worth trying any further.
-                                                       fail_channel_update = Some(msgs::HTLCFailChannelUpdate::NodeFailure {
+                                                       network_update = Some(NetworkUpdate::NodeFailure {
                                                                node_id: route_hop.pubkey,
                                                                is_permanent: true,
                                                        });
@@ -448,7 +449,7 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                                // TODO: Here (and a few other places) we assume that BADONION errors
                                                // are always "sourced" from the node previous to the one which failed
                                                // to decode the onion.
-                                               res = Some((fail_channel_update, !(error_code & PERM == PERM && is_from_final_node)));
+                                               res = Some((network_update, !(error_code & PERM == PERM && is_from_final_node)));
 
                                                let (description, title) = errors::get_onion_error_description(error_code);
                                                if debug_field_size > 0 && err_packet.failuremsg.len() >= 4 + debug_field_size {
@@ -460,7 +461,7 @@ pub(super) fn process_onion_failure<T: secp256k1::Signing, L: Deref>(secp_ctx: &
                                        } else {
                                                // Useless packet that we can't use but it passed HMAC, so it
                                                // definitely came from the peer in question
-                                               res = Some((Some(msgs::HTLCFailChannelUpdate::NodeFailure {
+                                               res = Some((Some(NetworkUpdate::NodeFailure {
                                                        node_id: route_hop.pubkey,
                                                        is_permanent: true,
                                                }), !is_from_final_node));
index 34bcda3a009a850b780bfffd6d4c087eb0256b28..1815d4a350cc0289100d207b9fa2a98029397b1d 100644 (file)
@@ -32,11 +32,11 @@ use routing::network_graph::NetGraphMsgHandler;
 use prelude::*;
 use io;
 use alloc::collections::LinkedList;
-use alloc::fmt::Debug;
 use sync::{Arc, Mutex};
 use core::sync::atomic::{AtomicUsize, Ordering};
 use core::{cmp, hash, fmt, mem};
 use core::ops::Deref;
+use core::convert::Infallible;
 #[cfg(feature = "std")] use std::error;
 
 use bitcoin::hashes::sha256::Hash as Sha256;
@@ -47,7 +47,7 @@ use bitcoin::hashes::{HashEngine, Hash};
 pub trait CustomMessageHandler: wire::CustomMessageReader {
        /// Called with the message type that was received and the buffer to be read.
        /// Can return a `MessageHandlingError` if the message could not be handled.
-       fn handle_custom_message(&self, msg: Self::CustomMessage) -> Result<(), LightningError>;
+       fn handle_custom_message(&self, msg: Self::CustomMessage, sender_node_id: &PublicKey) -> Result<(), LightningError>;
 
        /// Gets the list of pending messages which were generated by the custom message
        /// handler, clearing the list in the process. The first tuple element must
@@ -66,7 +66,6 @@ impl RoutingMessageHandler for IgnoringMessageHandler {
        fn handle_node_announcement(&self, _msg: &msgs::NodeAnnouncement) -> Result<bool, LightningError> { Ok(false) }
        fn handle_channel_announcement(&self, _msg: &msgs::ChannelAnnouncement) -> Result<bool, LightningError> { Ok(false) }
        fn handle_channel_update(&self, _msg: &msgs::ChannelUpdate) -> Result<bool, LightningError> { Ok(false) }
-       fn handle_htlc_fail_channel_update(&self, _update: &msgs::HTLCFailChannelUpdate) {}
        fn get_next_channel_announcements(&self, _starting_point: u64, _batch_amount: u8) ->
                Vec<(msgs::ChannelAnnouncement, Option<msgs::ChannelUpdate>, Option<msgs::ChannelUpdate>)> { Vec::new() }
        fn get_next_node_announcements(&self, _starting_point: Option<&PublicKey>, _batch_amount: u8) -> Vec<msgs::NodeAnnouncement> { Vec::new() }
@@ -81,28 +80,28 @@ impl Deref for IgnoringMessageHandler {
        fn deref(&self) -> &Self { self }
 }
 
-impl wire::Type for () {
+// Implement Type for Infallible, note that it cannot be constructed, and thus you can never call a
+// method that takes self for it.
+impl wire::Type for Infallible {
        fn type_id(&self) -> u16 {
-               // We should never call this for `DummyCustomType`
                unreachable!();
        }
 }
-
-impl Writeable for () {
+impl Writeable for Infallible {
        fn write<W: Writer>(&self, _: &mut W) -> Result<(), io::Error> {
                unreachable!();
        }
 }
 
 impl wire::CustomMessageReader for IgnoringMessageHandler {
-       type CustomMessage = ();
+       type CustomMessage = Infallible;
        fn read<R: io::Read>(&self, _message_type: u16, _buffer: &mut R) -> Result<Option<Self::CustomMessage>, msgs::DecodeError> {
                Ok(None)
        }
 }
 
 impl CustomMessageHandler for IgnoringMessageHandler {
-       fn handle_custom_message(&self, _msg: Self::CustomMessage) -> Result<(), LightningError> {
+       fn handle_custom_message(&self, _msg: Infallible, _sender_node_id: &PublicKey) -> Result<(), LightningError> {
                // Since we always return `None` in the read the handle method should never be called.
                unreachable!();
        }
@@ -470,7 +469,7 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, L: Deref, CMH: Deref> P
                CM::Target: ChannelMessageHandler,
                RM::Target: RoutingMessageHandler,
                L::Target: Logger,
-               CMH::Target: CustomMessageHandler + wire::CustomMessageReader {
+               CMH::Target: CustomMessageHandler {
        /// Constructs a new PeerManager with the given message handlers and node_id secret key
        /// ephemeral_random_data is used to derive per-connection ephemeral keys and must be
        /// cryptographically secure random bytes.
@@ -720,8 +719,8 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, L: Deref, CMH: Deref> P
        }
 
        /// Append a message to a peer's pending outbound/write buffer, and update the map of peers needing sends accordingly.
-       fn enqueue_message<M: wire::Type + Writeable + Debug>(&self, peer: &mut Peer, message: &M) {
-               let mut buffer = VecWriter(Vec::new());
+       fn enqueue_message<M: wire::Type>(&self, peer: &mut Peer, message: &M) {
+               let mut buffer = VecWriter(Vec::with_capacity(2048));
                wire::write(message, &mut buffer).unwrap(); // crash if the write failed
                let encoded_message = buffer.0;
 
@@ -1087,7 +1086,7 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, L: Deref, CMH: Deref> P
                                log_trace!(self.logger, "Received unknown odd message of type {}, ignoring", type_id);
                        },
                        wire::Message::Custom(custom) => {
-                               self.custom_message_handler.handle_custom_message(custom)?;
+                               self.custom_message_handler.handle_custom_message(custom, &peer.their_node_id.unwrap())?;
                        },
                };
                Ok(should_forward)
@@ -1170,6 +1169,9 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, L: Deref, CMH: Deref> P
        /// May call [`send_data`] on [`SocketDescriptor`]s. Thus, be very careful with reentrancy
        /// issues!
        ///
+       /// You don't have to call this function explicitly if you are using [`lightning-net-tokio`]
+       /// or one of the other clients provided in our language bindings.
+       ///
        /// [`send_payment`]: crate::ln::channelmanager::ChannelManager::send_payment
        /// [`ChannelManager::process_pending_htlc_forwards`]: crate::ln::channelmanager::ChannelManager::process_pending_htlc_forwards
        /// [`send_data`]: SocketDescriptor::send_data
@@ -1318,9 +1320,6 @@ impl<Descriptor: SocketDescriptor, CM: Deref, RM: Deref, L: Deref, CMH: Deref> P
                                                let peer = get_peer_for_forwarding!(node_id);
                                                peer.pending_outbound_buffer.push_back(peer.channel_encryptor.encrypt_message(&encode_msg!(msg)));
                                        },
-                                       MessageSendEvent::PaymentFailureNetworkUpdate { ref update } => {
-                                               self.message_handler.route_handler.handle_htlc_fail_channel_update(update);
-                                       },
                                        MessageSendEvent::HandleError { ref node_id, ref action } => {
                                                match *action {
                                                        msgs::ErrorAction::DisconnectPeer { ref msg } => {
index bbdb5bfac6bae9270e30bb99434c818020a3d475..aca67c8f9f0ebe40980777f0c24f14799b742660 100644 (file)
@@ -14,9 +14,10 @@ use chain::transaction::OutPoint;
 use chain::{Confirm, Watch};
 use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs};
 use ln::features::InitFeatures;
-use ln::msgs::{ChannelMessageHandler, ErrorAction, HTLCFailChannelUpdate};
+use ln::msgs::{ChannelMessageHandler, ErrorAction};
+use routing::network_graph::NetworkUpdate;
 use util::enforcing_trait_impls::EnforcingSigner;
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
 use util::test_utils;
 use util::ser::{ReadableArgs, Writeable};
 
@@ -80,6 +81,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
                connect_block(&nodes[2], &Block { header, txdata: node_1_commitment_txn.clone() });
                check_added_monitors!(nodes[2], 1);
                check_closed_broadcast!(nodes[2], true); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
+               check_closed_event!(nodes[2], 1, ClosureReason::CommitmentTxConfirmed);
                let node_2_commitment_txn = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap();
                assert_eq!(node_2_commitment_txn.len(), 3); // ChannelMonitor: 1 offered HTLC-Claim, ChannelManger: 1 local commitment tx, 1 Received HTLC-Claim
                assert_eq!(node_2_commitment_txn[1].output.len(), 2); // to-remote and Received HTLC (to-self is dust)
@@ -121,6 +123,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
        };
        check_added_monitors!(nodes[1], 1);
        check_closed_broadcast!(nodes[1], true); // We should get a BroadcastChannelUpdate (and *only* a BroadcstChannelUpdate)
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        // Connect ANTI_REORG_DELAY - 2 blocks, giving us a confirmation count of ANTI_REORG_DELAY - 1.
        connect_blocks(&nodes[1], ANTI_REORG_DELAY - 2);
        check_added_monitors!(nodes[1], 0);
@@ -163,12 +166,7 @@ fn do_test_onchain_htlc_reorg(local_commitment: bool, claim: bool) {
        if claim {
                expect_payment_sent!(nodes[0], our_payment_preimage);
        } else {
-               let events = nodes[0].node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), 1);
-               if let MessageSendEvent::PaymentFailureNetworkUpdate { update: HTLCFailChannelUpdate::ChannelClosed { ref is_permanent, .. } } = events[0] {
-                       assert!(is_permanent);
-               } else { panic!("Unexpected event!"); }
-               expect_payment_failed!(nodes[0], our_payment_hash, false);
+               expect_payment_failed_with_update!(nodes[0], our_payment_hash, false, chan_2.0.contents.short_channel_id, true);
        }
 }
 
@@ -302,6 +300,12 @@ fn do_test_unconf_chan(reload_node: bool, reorg_after_reload: bool, use_funding_
        *nodes[0].chain_monitor.expect_channel_force_closed.lock().unwrap() = Some((chan.2, true));
        nodes[0].node.test_process_background_events(); // Required to free the pending background monitor update
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
+       if connect_style == ConnectStyle::FullBlockViaListen && !use_funding_unconfirmed {
+               check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "Funding transaction was un-confirmed. Locked at 6 confs, now have 2 confs.".to_string() });
+       } else {
+               check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "Funding transaction was un-confirmed. Locked at 6 confs, now have 0 confs.".to_string() });
+       }
        assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
        nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 
@@ -373,6 +377,7 @@ fn test_set_outpoints_partial_claiming() {
        // Connect blocks on node A commitment transaction
        mine_transaction(&nodes[0], &remote_txn[0]);
        check_closed_broadcast!(nodes[0], true);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        check_added_monitors!(nodes[0], 1);
        // Verify node A broadcast tx claiming both HTLCs
        {
@@ -390,6 +395,7 @@ fn test_set_outpoints_partial_claiming() {
        // Connect blocks on node B
        connect_blocks(&nodes[1], 135);
        check_closed_broadcast!(nodes[1], true);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
        check_added_monitors!(nodes[1], 1);
        // Verify node B broadcast 2 HTLC-timeout txn
        let partial_claim_tx = {
@@ -464,9 +470,11 @@ fn do_test_to_remote_after_local_detection(style: ConnectStyle) {
        assert!(nodes[0].node.list_channels().is_empty());
        check_closed_broadcast!(nodes[0], true);
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::CommitmentTxConfirmed);
        assert!(nodes[1].node.list_channels().is_empty());
        check_closed_broadcast!(nodes[1], true);
        check_added_monitors!(nodes[1], 1);
+       check_closed_event!(nodes[1], 1, ClosureReason::CommitmentTxConfirmed);
 
        // Drop transactions broadcasted in response to the first commitment transaction (we have good
        // test coverage of these things already elsewhere).
index 4e81d76ad670d18b2c8b172d4acd94ea62c64607..00ad48bb1e31c07cf695f52282ea2b746cf05455 100644 (file)
@@ -3,7 +3,7 @@
 use bitcoin::blockdata::opcodes::all::OP_PUSHBYTES_0 as SEGWIT_V0;
 use bitcoin::blockdata::script::{Builder, Script};
 use bitcoin::hashes::Hash;
-use bitcoin::hash_types::{PubkeyHash, ScriptHash, WPubkeyHash, WScriptHash};
+use bitcoin::hash_types::{WPubkeyHash, WScriptHash};
 use bitcoin::secp256k1::key::PublicKey;
 
 use ln::features::InitFeatures;
@@ -66,16 +66,6 @@ impl ShutdownScript {
                Self(ShutdownScriptImpl::Legacy(pubkey))
        }
 
-       /// Generates a P2PKH script pubkey from the given [`PubkeyHash`].
-       pub fn new_p2pkh(pubkey_hash: &PubkeyHash) -> Self {
-               Self(ShutdownScriptImpl::Bolt2(Script::new_p2pkh(pubkey_hash)))
-       }
-
-       /// Generates a P2SH script pubkey from the given [`ScriptHash`].
-       pub fn new_p2sh(script_hash: &ScriptHash) -> Self {
-               Self(ShutdownScriptImpl::Bolt2(Script::new_p2sh(script_hash)))
-       }
-
        /// Generates a P2WPKH script pubkey from the given [`WPubkeyHash`].
        pub fn new_p2wpkh(pubkey_hash: &WPubkeyHash) -> Self {
                Self(ShutdownScriptImpl::Bolt2(Script::new_v0_wpkh(pubkey_hash)))
@@ -126,7 +116,9 @@ impl ShutdownScript {
        }
 }
 
-fn is_bolt2_compliant(script: &Script, features: &InitFeatures) -> bool {
+/// Check if a given script is compliant with BOLT 2's shutdown script requirements for the given
+/// counterparty features.
+pub(crate) fn is_bolt2_compliant(script: &Script, features: &InitFeatures) -> bool {
        if script.is_p2pkh() || script.is_p2sh() || script.is_v0_p2wpkh() || script.is_v0_p2wsh() {
                true
        } else if features.supports_shutdown_anysegwit() {
@@ -136,6 +128,8 @@ fn is_bolt2_compliant(script: &Script, features: &InitFeatures) -> bool {
        }
 }
 
+// Note that this is only for our own shutdown scripts. Counterparties are still allowed to send us
+// non-witness shutdown scripts which this rejects.
 impl TryFrom<Script> for ShutdownScript {
        type Error = InvalidShutdownScript;
 
@@ -144,11 +138,13 @@ impl TryFrom<Script> for ShutdownScript {
        }
 }
 
+// Note that this is only for our own shutdown scripts. Counterparties are still allowed to send us
+// non-witness shutdown scripts which this rejects.
 impl TryFrom<(Script, &InitFeatures)> for ShutdownScript {
        type Error = InvalidShutdownScript;
 
        fn try_from((script, features): (Script, &InitFeatures)) -> Result<Self, Self::Error> {
-               if is_bolt2_compliant(&script, features) {
+               if is_bolt2_compliant(&script, features) && script.is_witness_program() {
                        Ok(Self(ShutdownScriptImpl::Bolt2(script)))
                } else {
                        Err(InvalidShutdownScript { script })
@@ -216,30 +212,6 @@ mod shutdown_script_tests {
                assert_eq!(shutdown_script.into_inner(), p2wpkh_script);
        }
 
-       #[test]
-       fn generates_p2pkh_from_pubkey_hash() {
-               let pubkey_hash = pubkey().pubkey_hash();
-               let p2pkh_script = Script::new_p2pkh(&pubkey_hash);
-
-               let shutdown_script = ShutdownScript::new_p2pkh(&pubkey_hash);
-               assert!(shutdown_script.is_compatible(&InitFeatures::known()));
-               assert!(shutdown_script.is_compatible(&InitFeatures::known().clear_shutdown_anysegwit()));
-               assert_eq!(shutdown_script.into_inner(), p2pkh_script);
-               assert!(ShutdownScript::try_from(p2pkh_script).is_ok());
-       }
-
-       #[test]
-       fn generates_p2sh_from_script_hash() {
-               let script_hash = redeem_script().script_hash();
-               let p2sh_script = Script::new_p2sh(&script_hash);
-
-               let shutdown_script = ShutdownScript::new_p2sh(&script_hash);
-               assert!(shutdown_script.is_compatible(&InitFeatures::known()));
-               assert!(shutdown_script.is_compatible(&InitFeatures::known().clear_shutdown_anysegwit()));
-               assert_eq!(shutdown_script.into_inner(), p2sh_script);
-               assert!(ShutdownScript::try_from(p2sh_script).is_ok());
-       }
-
        #[test]
        fn generates_p2wpkh_from_pubkey_hash() {
                let pubkey_hash = pubkey().wpubkey_hash().unwrap();
index a40ad23761e36853015a8284e42ed6cbbcbad36d..d13b7d2df12fece9a5d78092a21caa73e816451b 100644 (file)
@@ -13,6 +13,7 @@ use chain::keysinterface::KeysInterface;
 use chain::transaction::OutPoint;
 use ln::{PaymentPreimage, PaymentHash};
 use ln::channelmanager::PaymentSendFailure;
+use routing::network_graph::NetworkUpdate;
 use routing::router::get_route;
 use ln::features::{InitFeatures, InvoiceFeatures};
 use ln::msgs;
@@ -20,7 +21,7 @@ use ln::msgs::{ChannelMessageHandler, ErrorAction};
 use ln::script::ShutdownScript;
 use util::test_utils;
 use util::test_utils::OnGetShutdownScriptpubkey;
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, ClosureReason};
 use util::errors::APIError;
 use util::config::UserConfig;
 
@@ -66,6 +67,8 @@ fn pre_funding_lock_shutdown_test() {
 
        assert!(nodes[0].node.list_channels().is_empty());
        assert!(nodes[1].node.list_channels().is_empty());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -94,8 +97,8 @@ fn updates_shutdown_wait() {
 
        let net_graph_msg_handler0 = &nodes[0].net_graph_msg_handler;
        let net_graph_msg_handler1 = &nodes[1].net_graph_msg_handler;
-       let route_1 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler0.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
-       let route_2 = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler1.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route_1 = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler0.network_graph, &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route_2 = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler1.network_graph, &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        unwrap_send_err!(nodes[0].node.send_payment(&route_1, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable {..}, {});
        unwrap_send_err!(nodes[1].node.send_payment(&route_2, payment_hash, &Some(payment_secret)), true, APIError::ChannelUnavailable {..}, {});
 
@@ -138,6 +141,8 @@ fn updates_shutdown_wait() {
        nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &node_0_2nd_closing_signed.unwrap());
        let (_, node_1_none) = get_closing_signed_broadcast!(nodes[1].node, nodes[0].node.get_our_node_id());
        assert!(node_1_none.is_none());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 
        assert!(nodes[0].node.list_channels().is_empty());
 
@@ -146,6 +151,8 @@ fn updates_shutdown_wait() {
        close_channel(&nodes[1], &nodes[2], &chan_2.2, chan_2.3, true);
        assert!(nodes[1].node.list_channels().is_empty());
        assert!(nodes[2].node.list_channels().is_empty());
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[2], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -161,7 +168,7 @@ fn htlc_fail_async_shutdown() {
 
        let (_, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[2]);
        let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
-       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph, &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
        nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(nodes[0], 1);
        let updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
@@ -192,17 +199,11 @@ fn htlc_fail_async_shutdown() {
        nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates_2.update_fail_htlcs[0]);
        commitment_signed_dance!(nodes[0], nodes[1], updates_2.commitment_signed, false, true);
 
-       expect_payment_failed!(nodes[0], our_payment_hash, false);
+       expect_payment_failed_with_update!(nodes[0], our_payment_hash, false, chan_2.0.contents.short_channel_id, true);
 
        let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
-       assert_eq!(msg_events.len(), 2);
-       match msg_events[0] {
-               MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
-                       assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
-               },
-               _ => panic!("Unexpected event"),
-       }
-       let node_0_closing_signed = match msg_events[1] {
+       assert_eq!(msg_events.len(), 1);
+       let node_0_closing_signed = match msg_events[0] {
                MessageSendEvent::SendClosingSigned { ref node_id, ref msg } => {
                        assert_eq!(*node_id, nodes[1].node.get_our_node_id());
                        (*msg).clone()
@@ -226,6 +227,9 @@ fn htlc_fail_async_shutdown() {
        close_channel(&nodes[1], &nodes[2], &chan_2.2, chan_2.3, true);
        assert!(nodes[1].node.list_channels().is_empty());
        assert!(nodes[2].node.list_channels().is_empty());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 2, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[2], 1, ClosureReason::CooperativeClosure);
 }
 
 fn do_test_shutdown_rebroadcast(recv_count: u8) {
@@ -364,6 +368,7 @@ fn do_test_shutdown_rebroadcast(recv_count: u8) {
                nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &node_0_2nd_closing_signed.unwrap());
                let (_, node_1_none) = get_closing_signed_broadcast!(nodes[1].node, nodes[0].node.get_our_node_id());
                assert!(node_1_none.is_none());
+               check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
        } else {
                // If one node, however, received + responded with an identical closing_signed we end
                // up erroring and node[0] will try to broadcast its own latest commitment transaction.
@@ -392,6 +397,7 @@ fn do_test_shutdown_rebroadcast(recv_count: u8) {
                // closing_signed so we do it ourselves
                check_closed_broadcast!(nodes[1], false);
                check_added_monitors!(nodes[1], 1);
+               check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "Failed to find corresponding channel".to_string() });
        }
 
        assert!(nodes[0].node.list_channels().is_empty());
@@ -401,6 +407,9 @@ fn do_test_shutdown_rebroadcast(recv_count: u8) {
        close_channel(&nodes[1], &nodes[2], &chan_2.2, chan_2.3, true);
        assert!(nodes[1].node.list_channels().is_empty());
        assert!(nodes[2].node.list_channels().is_empty());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[2], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -433,7 +442,8 @@ fn test_upfront_shutdown_script() {
        node_0_shutdown.scriptpubkey = Builder::new().push_opcode(opcodes::all::OP_RETURN).into_script().to_p2sh();
        // Test we enforce upfront_scriptpbukey if by providing a diffrent one at closing that  we disconnect peer
        nodes[2].node.handle_shutdown(&nodes[0].node.get_our_node_id(), &InitFeatures::known(), &node_0_shutdown);
-    assert!(regex::Regex::new(r"Got shutdown request with a scriptpubkey \([A-Fa-f0-9]+\) which did not match their previous scriptpubkey.").unwrap().is_match(check_closed_broadcast!(nodes[2], true).unwrap().data.as_str()));
+       assert!(regex::Regex::new(r"Got shutdown request with a scriptpubkey \([A-Fa-f0-9]+\) which did not match their previous scriptpubkey.").unwrap().is_match(check_closed_broadcast!(nodes[2], true).unwrap().data.as_str()));
+       check_closed_event!(nodes[2], 1, ClosureReason::ProcessingError { err: "Got shutdown request with a scriptpubkey (a91441c98a140039816273e50db317422c11c2bfcc8887) which did not match their previous scriptpubkey.".to_string() });
        check_added_monitors!(nodes[2], 1);
 
        // We test that in case of peer committing upfront to a script, if it doesn't change at closing, we sign
@@ -543,6 +553,7 @@ fn test_unsupported_anysegwit_upfront_shutdown_script() {
                },
                _ => panic!("Unexpected event"),
        }
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "Peer is signaling upfront_shutdown but has provided an unacceptable scriptpubkey format: Script(OP_PUSHNUM_16 OP_PUSHBYTES_2 0028)".to_string() });
 }
 
 #[test]
@@ -690,6 +701,7 @@ fn test_unsupported_anysegwit_shutdown_script() {
                _ => panic!("Unexpected event"),
        }
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "Got a nonstandard scriptpubkey (60020028) from remote peer".to_string() });
 }
 
 #[test]
@@ -725,6 +737,7 @@ fn test_invalid_shutdown_script() {
                _ => panic!("Unexpected event"),
        }
        check_added_monitors!(nodes[0], 1);
+       check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError { err: "Got a nonstandard scriptpubkey (00020000) from remote peer".to_string() });
 }
 
 #[derive(PartialEq)]
@@ -790,7 +803,9 @@ fn do_test_closing_signed_reinit_timeout(timeout_step: TimeoutStep) {
                let node_0_2nd_closing_signed = get_closing_signed_broadcast!(nodes[0].node, nodes[1].node.get_our_node_id());
                if timeout_step == TimeoutStep::NoTimeout {
                        nodes[1].node.handle_closing_signed(&nodes[0].node.get_our_node_id(), &node_0_2nd_closing_signed.1.unwrap());
+                       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
                }
+               check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
        }
 
        if timeout_step != TimeoutStep::NoTimeout {
@@ -813,6 +828,7 @@ fn do_test_closing_signed_reinit_timeout(timeout_step: TimeoutStep) {
                         txn[0].output[0].script_pubkey.is_v0_p2wsh()));
                check_closed_broadcast!(nodes[1], true);
                check_added_monitors!(nodes[1], 1);
+               check_closed_event!(nodes[1], 1, ClosureReason::ProcessingError { err: "closing_signed negotiation failed to finish within two timer ticks".to_string() });
        } else {
                assert!(txn[0].output[0].script_pubkey.is_v0_p2wpkh());
                assert!(txn[0].output[1].script_pubkey.is_v0_p2wpkh());
@@ -872,6 +888,8 @@ fn do_simple_legacy_shutdown_test(high_initiator_fee: bool) {
        nodes[0].node.handle_closing_signed(&nodes[1].node.get_our_node_id(), &node_1_closing_signed.unwrap());
        let (_, node_0_none) = get_closing_signed_broadcast!(nodes[0].node, nodes[1].node.get_our_node_id());
        assert!(node_0_none.is_none());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
 
 #[test]
@@ -925,4 +943,6 @@ fn simple_target_feerate_shutdown() {
        nodes[0].node.handle_closing_signed(&nodes[1].node.get_our_node_id(), &node_1_closing_signed);
        let (_, node_0_none) = get_closing_signed_broadcast!(nodes[0].node, nodes[1].node.get_our_node_id());
        assert!(node_0_none.is_none());
+       check_closed_event!(nodes[0], 1, ClosureReason::CooperativeClosure);
+       check_closed_event!(nodes[1], 1, ClosureReason::CooperativeClosure);
 }
index 4caf3543b0f9f02e5ec3b650c8c9a82fa9cc861c..d3cc257f5a6b18cb1776475b4fec69949521e1d6 100644 (file)
@@ -20,7 +20,7 @@ use util::ser::{Readable, Writeable, Writer};
 /// decoders.
 pub trait CustomMessageReader {
        /// The type of the message decoded by the implementation.
-       type CustomMessage: core::fmt::Debug + Type + Writeable;
+       type CustomMessage: Type;
        /// Decodes a custom message to `CustomMessageType`. If the given message type is known to the
        /// implementation and the message could be decoded, must return `Ok(Some(message))`. If the
        /// message type is unknown to the implementation, must return `Ok(None)`. If a decoding error
@@ -245,12 +245,12 @@ pub(crate) use self::encode::Encode;
 /// Defines a type identifier for sending messages over the wire.
 ///
 /// Messages implementing this trait specify a type and must be [`Writeable`].
-pub trait Type {
+pub trait Type: core::fmt::Debug + Writeable {
        /// Returns the type identifying the message payload.
        fn type_id(&self) -> u16;
 }
 
-impl<T> Type for T where T: Encode {
+impl<T: core::fmt::Debug + Writeable> Type for T where T: Encode {
        fn type_id(&self) -> u16 {
                T::TYPE
        }
@@ -457,6 +457,10 @@ mod tests {
                }
        }
 
+       impl Type for () {
+               fn type_id(&self) -> u16 { unreachable!(); }
+       }
+
        #[test]
        fn is_even_message_type() {
                let message = Message::<()>::Unknown(42);
index 8accdab60882321dea801497643478b702a21b47..bdb305e7870a2b9a30e9236c34b516dc2328c873 100644 (file)
@@ -29,7 +29,7 @@ use ln::msgs::{QueryChannelRange, ReplyChannelRange, QueryShortChannelIds, Reply
 use ln::msgs;
 use util::ser::{Writeable, Readable, Writer};
 use util::logger::{Logger, Level};
-use util::events::{MessageSendEvent, MessageSendEventsProvider};
+use util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider};
 use util::scid_utils::{block_from_scid, scid_from_parts, MAX_SCID_BLOCK};
 
 use io;
@@ -51,56 +51,120 @@ const MAX_EXCESS_BYTES_FOR_RELAY: usize = 1024;
 const MAX_SCIDS_PER_REPLY: usize = 8000;
 
 /// Represents the network as nodes and channels between them
-#[derive(Clone, PartialEq)]
 pub struct NetworkGraph {
        genesis_hash: BlockHash,
-       channels: BTreeMap<u64, ChannelInfo>,
-       nodes: BTreeMap<PublicKey, NodeInfo>,
+       // Lock order: channels -> nodes
+       channels: RwLock<BTreeMap<u64, ChannelInfo>>,
+       nodes: RwLock<BTreeMap<PublicKey, NodeInfo>>,
 }
 
-/// A simple newtype for RwLockReadGuard<'a, NetworkGraph>.
-/// This exists only to make accessing a RwLock<NetworkGraph> possible from
-/// the C bindings, as it can be done directly in Rust code.
-pub struct LockedNetworkGraph<'a>(pub RwLockReadGuard<'a, NetworkGraph>);
+impl Clone for NetworkGraph {
+       fn clone(&self) -> Self {
+               let channels = self.channels.read().unwrap();
+               let nodes = self.nodes.read().unwrap();
+               Self {
+                       genesis_hash: self.genesis_hash.clone(),
+                       channels: RwLock::new(channels.clone()),
+                       nodes: RwLock::new(nodes.clone()),
+               }
+       }
+}
+
+/// A read-only view of [`NetworkGraph`].
+pub struct ReadOnlyNetworkGraph<'a> {
+       channels: RwLockReadGuard<'a, BTreeMap<u64, ChannelInfo>>,
+       nodes: RwLockReadGuard<'a, BTreeMap<PublicKey, NodeInfo>>,
+}
+
+/// Update to the [`NetworkGraph`] based on payment failure information conveyed via the Onion
+/// return packet by a node along the route. See [BOLT #4] for details.
+///
+/// [BOLT #4]: https://github.com/lightningnetwork/lightning-rfc/blob/master/04-onion-routing.md
+#[derive(Clone, Debug, PartialEq)]
+pub enum NetworkUpdate {
+       /// An error indicating a `channel_update` messages should be applied via
+       /// [`NetworkGraph::update_channel`].
+       ChannelUpdateMessage {
+               /// The update to apply via [`NetworkGraph::update_channel`].
+               msg: ChannelUpdate,
+       },
+       /// An error indicating only that a channel has been closed, which should be applied via
+       /// [`NetworkGraph::close_channel_from_update`].
+       ChannelClosed {
+               /// The short channel id of the closed channel.
+               short_channel_id: u64,
+               /// Whether the channel should be permanently removed or temporarily disabled until a new
+               /// `channel_update` message is received.
+               is_permanent: bool,
+       },
+       /// An error indicating only that a node has failed, which should be applied via
+       /// [`NetworkGraph::fail_node`].
+       NodeFailure {
+               /// The node id of the failed node.
+               node_id: PublicKey,
+               /// Whether the node should be permanently removed from consideration or can be restored
+               /// when a new `channel_update` message is received.
+               is_permanent: bool,
+       }
+}
+
+impl_writeable_tlv_based_enum_upgradable!(NetworkUpdate,
+       (0, ChannelUpdateMessage) => {
+               (0, msg, required),
+       },
+       (2, ChannelClosed) => {
+               (0, short_channel_id, required),
+               (2, is_permanent, required),
+       },
+       (4, NodeFailure) => {
+               (0, node_id, required),
+               (2, is_permanent, required),
+       },
+);
+
+impl<C: Deref, L: Deref> EventHandler for NetGraphMsgHandler<C, L>
+where C::Target: chain::Access, L::Target: Logger {
+       fn handle_event(&self, event: &Event) {
+               if let Event::PaymentPathFailed { payment_hash: _, rejected_by_dest: _, network_update, .. } = event {
+                       if let Some(network_update) = network_update {
+                               self.handle_network_update(network_update);
+                       }
+               }
+       }
+}
 
 /// Receives and validates network updates from peers,
 /// stores authentic and relevant data as a network graph.
 /// This network graph is then used for routing payments.
 /// Provides interface to help with initial routing sync by
 /// serving historical announcements.
-pub struct NetGraphMsgHandler<C: Deref, L: Deref> where C::Target: chain::Access, L::Target: Logger {
+///
+/// Serves as an [`EventHandler`] for applying updates from [`Event::PaymentPathFailed`] to the
+/// [`NetworkGraph`].
+pub struct NetGraphMsgHandler<C: Deref, L: Deref>
+where C::Target: chain::Access, L::Target: Logger
+{
        secp_ctx: Secp256k1<secp256k1::VerifyOnly>,
        /// Representation of the payment channel network
-       pub network_graph: RwLock<NetworkGraph>,
+       pub network_graph: NetworkGraph,
        chain_access: Option<C>,
        full_syncs_requested: AtomicUsize,
        pending_events: Mutex<Vec<MessageSendEvent>>,
        logger: L,
 }
 
-impl<C: Deref, L: Deref> NetGraphMsgHandler<C, L> where C::Target: chain::Access, L::Target: Logger {
+impl<C: Deref, L: Deref> NetGraphMsgHandler<C, L>
+where C::Target: chain::Access, L::Target: Logger
+{
        /// Creates a new tracker of the actual state of the network of channels and nodes,
-       /// assuming a fresh network graph.
+       /// assuming an existing Network Graph.
        /// Chain monitor is used to make sure announced channels exist on-chain,
        /// channel data is correct, and that the announcement is signed with
        /// channel owners' keys.
-       pub fn new(genesis_hash: BlockHash, chain_access: Option<C>, logger: L) -> Self {
+       pub fn new(network_graph: NetworkGraph, chain_access: Option<C>, logger: L) -> Self {
                NetGraphMsgHandler {
                        secp_ctx: Secp256k1::verification_only(),
-                       network_graph: RwLock::new(NetworkGraph::new(genesis_hash)),
-                       full_syncs_requested: AtomicUsize::new(0),
-                       chain_access,
-                       pending_events: Mutex::new(vec![]),
-                       logger,
-               }
-       }
-
-       /// Creates a new tracker of the actual state of the network of channels and nodes,
-       /// assuming an existing Network Graph.
-       pub fn from_net_graph(chain_access: Option<C>, logger: L, network_graph: NetworkGraph) -> Self {
-               NetGraphMsgHandler {
-                       secp_ctx: Secp256k1::verification_only(),
-                       network_graph: RwLock::new(network_graph),
+                       network_graph,
                        full_syncs_requested: AtomicUsize::new(0),
                        chain_access,
                        pending_events: Mutex::new(vec![]),
@@ -115,14 +179,6 @@ impl<C: Deref, L: Deref> NetGraphMsgHandler<C, L> where C::Target: chain::Access
                self.chain_access = chain_access;
        }
 
-       /// Take a read lock on the network_graph and return it in the C-bindings
-       /// newtype helper. This is likely only useful when called via the C
-       /// bindings as you can call `self.network_graph.read().unwrap()` in Rust
-       /// yourself.
-       pub fn read_locked_graph<'a>(&'a self) -> LockedNetworkGraph<'a> {
-               LockedNetworkGraph(self.network_graph.read().unwrap())
-       }
-
        /// Returns true when a full routing table sync should be performed with a peer.
        fn should_request_full_sync(&self, _node_id: &PublicKey) -> bool {
                //TODO: Determine whether to request a full sync based on the network map.
@@ -134,16 +190,31 @@ impl<C: Deref, L: Deref> NetGraphMsgHandler<C, L> where C::Target: chain::Access
                        false
                }
        }
-}
 
-impl<'a> LockedNetworkGraph<'a> {
-       /// Get a reference to the NetworkGraph which this read-lock contains.
-       pub fn graph(&self) -> &NetworkGraph {
-               &*self.0
+       /// Applies changes to the [`NetworkGraph`] from the given update.
+       fn handle_network_update(&self, update: &NetworkUpdate) {
+               match *update {
+                       NetworkUpdate::ChannelUpdateMessage { ref msg } => {
+                               let short_channel_id = msg.contents.short_channel_id;
+                               let is_enabled = msg.contents.flags & (1 << 1) != (1 << 1);
+                               let status = if is_enabled { "enabled" } else { "disabled" };
+                               log_debug!(self.logger, "Updating channel with channel_update from a payment failure. Channel {} is {}.", short_channel_id, status);
+                               let _ = self.network_graph.update_channel(msg, &self.secp_ctx);
+                       },
+                       NetworkUpdate::ChannelClosed { short_channel_id, is_permanent } => {
+                               let action = if is_permanent { "Removing" } else { "Disabling" };
+                               log_debug!(self.logger, "{} channel graph entry for {} due to a payment failure.", action, short_channel_id);
+                               self.network_graph.close_channel_from_update(short_channel_id, is_permanent);
+                       },
+                       NetworkUpdate::NodeFailure { ref node_id, is_permanent } => {
+                               let action = if is_permanent { "Removing" } else { "Disabling" };
+                               log_debug!(self.logger, "{} node graph entry for {} due to a payment failure.", action, node_id);
+                               self.network_graph.fail_node(node_id, is_permanent);
+                       },
+               }
        }
 }
 
-
 macro_rules! secp_verify_sig {
        ( $secp_ctx: expr, $msg: expr, $sig: expr, $pubkey: expr ) => {
                match $secp_ctx.verify($msg, $sig, $pubkey) {
@@ -153,47 +224,31 @@ macro_rules! secp_verify_sig {
        };
 }
 
-impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> where C::Target: chain::Access, L::Target: Logger {
+impl<C: Deref, L: Deref> RoutingMessageHandler for NetGraphMsgHandler<C, L>
+where C::Target: chain::Access, L::Target: Logger
+{
        fn handle_node_announcement(&self, msg: &msgs::NodeAnnouncement) -> Result<bool, LightningError> {
-               self.network_graph.write().unwrap().update_node_from_announcement(msg, &self.secp_ctx)?;
+               self.network_graph.update_node_from_announcement(msg, &self.secp_ctx)?;
                Ok(msg.contents.excess_data.len() <=  MAX_EXCESS_BYTES_FOR_RELAY &&
                   msg.contents.excess_address_data.len() <= MAX_EXCESS_BYTES_FOR_RELAY &&
                   msg.contents.excess_data.len() + msg.contents.excess_address_data.len() <= MAX_EXCESS_BYTES_FOR_RELAY)
        }
 
        fn handle_channel_announcement(&self, msg: &msgs::ChannelAnnouncement) -> Result<bool, LightningError> {
-               self.network_graph.write().unwrap().update_channel_from_announcement(msg, &self.chain_access, &self.secp_ctx)?;
+               self.network_graph.update_channel_from_announcement(msg, &self.chain_access, &self.secp_ctx)?;
                log_trace!(self.logger, "Added channel_announcement for {}{}", msg.contents.short_channel_id, if !msg.contents.excess_data.is_empty() { " with excess uninterpreted data!" } else { "" });
                Ok(msg.contents.excess_data.len() <= MAX_EXCESS_BYTES_FOR_RELAY)
        }
 
-       fn handle_htlc_fail_channel_update(&self, update: &msgs::HTLCFailChannelUpdate) {
-               match update {
-                       &msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg } => {
-                               let chan_enabled = msg.contents.flags & (1 << 1) != (1 << 1);
-                               log_debug!(self.logger, "Updating channel with channel_update from a payment failure. Channel {} is {}abled.", msg.contents.short_channel_id, if chan_enabled { "en" } else { "dis" });
-                               let _ = self.network_graph.write().unwrap().update_channel(msg, &self.secp_ctx);
-                       },
-                       &msgs::HTLCFailChannelUpdate::ChannelClosed { short_channel_id, is_permanent } => {
-                               log_debug!(self.logger, "{} channel graph entry for {} due to a payment failure.", if is_permanent { "Removing" } else { "Disabling" }, short_channel_id);
-                               self.network_graph.write().unwrap().close_channel_from_update(short_channel_id, is_permanent);
-                       },
-                       &msgs::HTLCFailChannelUpdate::NodeFailure { ref node_id, is_permanent } => {
-                               log_debug!(self.logger, "{} node graph entry for {} due to a payment failure.", if is_permanent { "Removing" } else { "Disabling" }, node_id);
-                               self.network_graph.write().unwrap().fail_node(node_id, is_permanent);
-                       },
-               }
-       }
-
        fn handle_channel_update(&self, msg: &msgs::ChannelUpdate) -> Result<bool, LightningError> {
-               self.network_graph.write().unwrap().update_channel(msg, &self.secp_ctx)?;
+               self.network_graph.update_channel(msg, &self.secp_ctx)?;
                Ok(msg.contents.excess_data.len() <= MAX_EXCESS_BYTES_FOR_RELAY)
        }
 
        fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(ChannelAnnouncement, Option<ChannelUpdate>, Option<ChannelUpdate>)> {
-               let network_graph = self.network_graph.read().unwrap();
                let mut result = Vec::with_capacity(batch_amount as usize);
-               let mut iter = network_graph.get_channels().range(starting_point..);
+               let channels = self.network_graph.channels.read().unwrap();
+               let mut iter = channels.range(starting_point..);
                while result.len() < batch_amount as usize {
                        if let Some((_, ref chan)) = iter.next() {
                                if chan.announcement_message.is_some() {
@@ -219,14 +274,14 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
        }
 
        fn get_next_node_announcements(&self, starting_point: Option<&PublicKey>, batch_amount: u8) -> Vec<NodeAnnouncement> {
-               let network_graph = self.network_graph.read().unwrap();
                let mut result = Vec::with_capacity(batch_amount as usize);
+               let nodes = self.network_graph.nodes.read().unwrap();
                let mut iter = if let Some(pubkey) = starting_point {
-                               let mut iter = network_graph.get_nodes().range((*pubkey)..);
+                               let mut iter = nodes.range((*pubkey)..);
                                iter.next();
                                iter
                        } else {
-                               network_graph.get_nodes().range(..)
+                               nodes.range(..)
                        };
                while result.len() < batch_amount as usize {
                        if let Some((_, ref node)) = iter.next() {
@@ -270,7 +325,7 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
                pending_events.push(MessageSendEvent::SendChannelRangeQuery {
                        node_id: their_node_id.clone(),
                        msg: QueryChannelRange {
-                               chain_hash: self.network_graph.read().unwrap().genesis_hash,
+                               chain_hash: self.network_graph.genesis_hash,
                                first_blocknum,
                                number_of_blocks,
                        },
@@ -332,8 +387,6 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
        fn handle_query_channel_range(&self, their_node_id: &PublicKey, msg: QueryChannelRange) -> Result<(), LightningError> {
                log_debug!(self.logger, "Handling query_channel_range peer={}, first_blocknum={}, number_of_blocks={}", log_pubkey!(their_node_id), msg.first_blocknum, msg.number_of_blocks);
 
-               let network_graph = self.network_graph.read().unwrap();
-
                let inclusive_start_scid = scid_from_parts(msg.first_blocknum as u64, 0, 0);
 
                // We might receive valid queries with end_blocknum that would overflow SCID conversion.
@@ -341,7 +394,7 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
                let exclusive_end_scid = scid_from_parts(cmp::min(msg.end_blocknum() as u64, MAX_SCID_BLOCK), 0, 0);
 
                // Per spec, we must reply to a query. Send an empty message when things are invalid.
-               if msg.chain_hash != network_graph.genesis_hash || inclusive_start_scid.is_err() || exclusive_end_scid.is_err() || msg.number_of_blocks == 0 {
+               if msg.chain_hash != self.network_graph.genesis_hash || inclusive_start_scid.is_err() || exclusive_end_scid.is_err() || msg.number_of_blocks == 0 {
                        let mut pending_events = self.pending_events.lock().unwrap();
                        pending_events.push(MessageSendEvent::SendReplyChannelRange {
                                node_id: their_node_id.clone(),
@@ -363,7 +416,8 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
                // (has at least one update). A peer may still want to know the channel
                // exists even if its not yet routable.
                let mut batches: Vec<Vec<u64>> = vec![Vec::with_capacity(MAX_SCIDS_PER_REPLY)];
-               for (_, ref chan) in network_graph.get_channels().range(inclusive_start_scid.unwrap()..exclusive_end_scid.unwrap()) {
+               let channels = self.network_graph.channels.read().unwrap();
+               for (_, ref chan) in channels.range(inclusive_start_scid.unwrap()..exclusive_end_scid.unwrap()) {
                        if let Some(chan_announcement) = &chan.announcement_message {
                                // Construct a new batch if last one is full
                                if batches.last().unwrap().len() == batches.last().unwrap().capacity() {
@@ -374,7 +428,7 @@ impl<C: Deref , L: Deref > RoutingMessageHandler for NetGraphMsgHandler<C, L> wh
                                batch.push(chan_announcement.contents.short_channel_id);
                        }
                }
-               drop(network_graph);
+               drop(channels);
 
                let mut pending_events = self.pending_events.lock().unwrap();
                let batch_count = batches.len();
@@ -616,13 +670,15 @@ impl Writeable for NetworkGraph {
                write_ver_prefix!(writer, SERIALIZATION_VERSION, MIN_SERIALIZATION_VERSION);
 
                self.genesis_hash.write(writer)?;
-               (self.channels.len() as u64).write(writer)?;
-               for (ref chan_id, ref chan_info) in self.channels.iter() {
+               let channels = self.channels.read().unwrap();
+               (channels.len() as u64).write(writer)?;
+               for (ref chan_id, ref chan_info) in channels.iter() {
                        (*chan_id).write(writer)?;
                        chan_info.write(writer)?;
                }
-               (self.nodes.len() as u64).write(writer)?;
-               for (ref node_id, ref node_info) in self.nodes.iter() {
+               let nodes = self.nodes.read().unwrap();
+               (nodes.len() as u64).write(writer)?;
+               for (ref node_id, ref node_info) in nodes.iter() {
                        node_id.write(writer)?;
                        node_info.write(writer)?;
                }
@@ -655,8 +711,8 @@ impl Readable for NetworkGraph {
 
                Ok(NetworkGraph {
                        genesis_hash,
-                       channels,
-                       nodes,
+                       channels: RwLock::new(channels),
+                       nodes: RwLock::new(nodes),
                })
        }
 }
@@ -664,47 +720,42 @@ impl Readable for NetworkGraph {
 impl fmt::Display for NetworkGraph {
        fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
                writeln!(f, "Network map\n[Channels]")?;
-               for (key, val) in self.channels.iter() {
+               for (key, val) in self.channels.read().unwrap().iter() {
                        writeln!(f, " {}: {}", key, val)?;
                }
                writeln!(f, "[Nodes]")?;
-               for (key, val) in self.nodes.iter() {
+               for (key, val) in self.nodes.read().unwrap().iter() {
                        writeln!(f, " {}: {}", log_pubkey!(key), val)?;
                }
                Ok(())
        }
 }
 
-impl NetworkGraph {
-       /// Returns all known valid channels' short ids along with announced channel info.
-       ///
-       /// (C-not exported) because we have no mapping for `BTreeMap`s
-       pub fn get_channels<'a>(&'a self) -> &'a BTreeMap<u64, ChannelInfo> { &self.channels }
-       /// Returns all known nodes' public keys along with announced node info.
-       ///
-       /// (C-not exported) because we have no mapping for `BTreeMap`s
-       pub fn get_nodes<'a>(&'a self) -> &'a BTreeMap<PublicKey, NodeInfo> { &self.nodes }
-
-       /// Get network addresses by node id.
-       /// Returns None if the requested node is completely unknown,
-       /// or if node announcement for the node was never received.
-       ///
-       /// (C-not exported) as there is no practical way to track lifetimes of returned values.
-       pub fn get_addresses<'a>(&'a self, pubkey: &PublicKey) -> Option<&'a Vec<NetAddress>> {
-               if let Some(node) = self.nodes.get(pubkey) {
-                       if let Some(node_info) = node.announcement_info.as_ref() {
-                               return Some(&node_info.addresses)
-                       }
-               }
-               None
+impl PartialEq for NetworkGraph {
+       fn eq(&self, other: &Self) -> bool {
+               self.genesis_hash == other.genesis_hash &&
+                       *self.channels.read().unwrap() == *other.channels.read().unwrap() &&
+                       *self.nodes.read().unwrap() == *other.nodes.read().unwrap()
        }
+}
 
+impl NetworkGraph {
        /// Creates a new, empty, network graph.
        pub fn new(genesis_hash: BlockHash) -> NetworkGraph {
                Self {
                        genesis_hash,
-                       channels: BTreeMap::new(),
-                       nodes: BTreeMap::new(),
+                       channels: RwLock::new(BTreeMap::new()),
+                       nodes: RwLock::new(BTreeMap::new()),
+               }
+       }
+
+       /// Returns a read-only view of the network graph.
+       pub fn read_only(&'_ self) -> ReadOnlyNetworkGraph<'_> {
+               let channels = self.channels.read().unwrap();
+               let nodes = self.nodes.read().unwrap();
+               ReadOnlyNetworkGraph {
+                       channels,
+                       nodes,
                }
        }
 
@@ -714,7 +765,7 @@ impl NetworkGraph {
        /// You probably don't want to call this directly, instead relying on a NetGraphMsgHandler's
        /// RoutingMessageHandler implementation to call it indirectly. This may be useful to accept
        /// routing messages from a source using a protocol other than the lightning P2P protocol.
-       pub fn update_node_from_announcement<T: secp256k1::Verification>(&mut self, msg: &msgs::NodeAnnouncement, secp_ctx: &Secp256k1<T>) -> Result<(), LightningError> {
+       pub fn update_node_from_announcement<T: secp256k1::Verification>(&self, msg: &msgs::NodeAnnouncement, secp_ctx: &Secp256k1<T>) -> Result<(), LightningError> {
                let msg_hash = hash_to_message!(&Sha256dHash::hash(&msg.contents.encode()[..])[..]);
                secp_verify_sig!(secp_ctx, &msg_hash, &msg.signature, &msg.contents.node_id);
                self.update_node_from_announcement_intern(&msg.contents, Some(&msg))
@@ -724,12 +775,12 @@ impl NetworkGraph {
        /// given node announcement without verifying the associated signatures. Because we aren't
        /// given the associated signatures here we cannot relay the node announcement to any of our
        /// peers.
-       pub fn update_node_from_unsigned_announcement(&mut self, msg: &msgs::UnsignedNodeAnnouncement) -> Result<(), LightningError> {
+       pub fn update_node_from_unsigned_announcement(&self, msg: &msgs::UnsignedNodeAnnouncement) -> Result<(), LightningError> {
                self.update_node_from_announcement_intern(msg, None)
        }
 
-       fn update_node_from_announcement_intern(&mut self, msg: &msgs::UnsignedNodeAnnouncement, full_msg: Option<&msgs::NodeAnnouncement>) -> Result<(), LightningError> {
-               match self.nodes.get_mut(&msg.node_id) {
+       fn update_node_from_announcement_intern(&self, msg: &msgs::UnsignedNodeAnnouncement, full_msg: Option<&msgs::NodeAnnouncement>) -> Result<(), LightningError> {
+               match self.nodes.write().unwrap().get_mut(&msg.node_id) {
                        None => Err(LightningError{err: "No existing channels for node_announcement".to_owned(), action: ErrorAction::IgnoreError}),
                        Some(node) => {
                                if let Some(node_info) = node.announcement_info.as_ref() {
@@ -764,10 +815,12 @@ impl NetworkGraph {
        ///
        /// If a `chain::Access` object is provided via `chain_access`, it will be called to verify
        /// the corresponding UTXO exists on chain and is correctly-formatted.
-       pub fn update_channel_from_announcement<T: secp256k1::Verification, C: Deref>
-                       (&mut self, msg: &msgs::ChannelAnnouncement, chain_access: &Option<C>, secp_ctx: &Secp256k1<T>)
-                       -> Result<(), LightningError>
-                       where C::Target: chain::Access {
+       pub fn update_channel_from_announcement<T: secp256k1::Verification, C: Deref>(
+               &self, msg: &msgs::ChannelAnnouncement, chain_access: &Option<C>, secp_ctx: &Secp256k1<T>
+       ) -> Result<(), LightningError>
+       where
+               C::Target: chain::Access,
+       {
                let msg_hash = hash_to_message!(&Sha256dHash::hash(&msg.contents.encode()[..])[..]);
                secp_verify_sig!(secp_ctx, &msg_hash, &msg.node_signature_1, &msg.contents.node_id_1);
                secp_verify_sig!(secp_ctx, &msg_hash, &msg.node_signature_2, &msg.contents.node_id_2);
@@ -782,17 +835,21 @@ impl NetworkGraph {
        ///
        /// If a `chain::Access` object is provided via `chain_access`, it will be called to verify
        /// the corresponding UTXO exists on chain and is correctly-formatted.
-       pub fn update_channel_from_unsigned_announcement<C: Deref>
-                       (&mut self, msg: &msgs::UnsignedChannelAnnouncement, chain_access: &Option<C>)
-                       -> Result<(), LightningError>
-                       where C::Target: chain::Access {
+       pub fn update_channel_from_unsigned_announcement<C: Deref>(
+               &self, msg: &msgs::UnsignedChannelAnnouncement, chain_access: &Option<C>
+       ) -> Result<(), LightningError>
+       where
+               C::Target: chain::Access,
+       {
                self.update_channel_from_unsigned_announcement_intern(msg, None, chain_access)
        }
 
-       fn update_channel_from_unsigned_announcement_intern<C: Deref>
-                       (&mut self, msg: &msgs::UnsignedChannelAnnouncement, full_msg: Option<&msgs::ChannelAnnouncement>, chain_access: &Option<C>)
-                       -> Result<(), LightningError>
-                       where C::Target: chain::Access {
+       fn update_channel_from_unsigned_announcement_intern<C: Deref>(
+               &self, msg: &msgs::UnsignedChannelAnnouncement, full_msg: Option<&msgs::ChannelAnnouncement>, chain_access: &Option<C>
+       ) -> Result<(), LightningError>
+       where
+               C::Target: chain::Access,
+       {
                if msg.node_id_1 == msg.node_id_2 || msg.bitcoin_key_1 == msg.bitcoin_key_2 {
                        return Err(LightningError{err: "Channel announcement node had a channel with itself".to_owned(), action: ErrorAction::IgnoreError});
                }
@@ -838,7 +895,9 @@ impl NetworkGraph {
                                        { full_msg.cloned() } else { None },
                        };
 
-               match self.channels.entry(msg.short_channel_id) {
+               let mut channels = self.channels.write().unwrap();
+               let mut nodes = self.nodes.write().unwrap();
+               match channels.entry(msg.short_channel_id) {
                        BtreeEntry::Occupied(mut entry) => {
                                //TODO: because asking the blockchain if short_channel_id is valid is only optional
                                //in the blockchain API, we need to handle it smartly here, though it's unclear
@@ -852,7 +911,7 @@ impl NetworkGraph {
                                        // b) we don't track UTXOs of channels we know about and remove them if they
                                        //    get reorg'd out.
                                        // c) it's unclear how to do so without exposing ourselves to massive DoS risk.
-                                       Self::remove_channel_in_nodes(&mut self.nodes, &entry.get(), msg.short_channel_id);
+                                       Self::remove_channel_in_nodes(&mut nodes, &entry.get(), msg.short_channel_id);
                                        *entry.get_mut() = chan_info;
                                } else {
                                        return Err(LightningError{err: "Already have knowledge of channel".to_owned(), action: ErrorAction::IgnoreAndLog(Level::Trace)})
@@ -865,7 +924,7 @@ impl NetworkGraph {
 
                macro_rules! add_channel_to_node {
                        ( $node_id: expr ) => {
-                               match self.nodes.entry($node_id) {
+                               match nodes.entry($node_id) {
                                        BtreeEntry::Occupied(node_entry) => {
                                                node_entry.into_mut().channels.push(msg.short_channel_id);
                                        },
@@ -890,13 +949,15 @@ impl NetworkGraph {
        /// If permanent, removes a channel from the local storage.
        /// May cause the removal of nodes too, if this was their last channel.
        /// If not permanent, makes channels unavailable for routing.
-       pub fn close_channel_from_update(&mut self, short_channel_id: u64, is_permanent: bool) {
+       pub fn close_channel_from_update(&self, short_channel_id: u64, is_permanent: bool) {
+               let mut channels = self.channels.write().unwrap();
                if is_permanent {
-                       if let Some(chan) = self.channels.remove(&short_channel_id) {
-                               Self::remove_channel_in_nodes(&mut self.nodes, &chan, short_channel_id);
+                       if let Some(chan) = channels.remove(&short_channel_id) {
+                               let mut nodes = self.nodes.write().unwrap();
+                               Self::remove_channel_in_nodes(&mut nodes, &chan, short_channel_id);
                        }
                } else {
-                       if let Some(chan) = self.channels.get_mut(&short_channel_id) {
+                       if let Some(chan) = channels.get_mut(&short_channel_id) {
                                if let Some(one_to_two) = chan.one_to_two.as_mut() {
                                        one_to_two.enabled = false;
                                }
@@ -907,7 +968,8 @@ impl NetworkGraph {
                }
        }
 
-       fn fail_node(&mut self, _node_id: &PublicKey, is_permanent: bool) {
+       /// Marks a node in the graph as failed.
+       pub fn fail_node(&self, _node_id: &PublicKey, is_permanent: bool) {
                if is_permanent {
                        // TODO: Wholly remove the node
                } else {
@@ -921,23 +983,24 @@ impl NetworkGraph {
        /// You probably don't want to call this directly, instead relying on a NetGraphMsgHandler's
        /// RoutingMessageHandler implementation to call it indirectly. This may be useful to accept
        /// routing messages from a source using a protocol other than the lightning P2P protocol.
-       pub fn update_channel<T: secp256k1::Verification>(&mut self, msg: &msgs::ChannelUpdate, secp_ctx: &Secp256k1<T>) -> Result<(), LightningError> {
+       pub fn update_channel<T: secp256k1::Verification>(&self, msg: &msgs::ChannelUpdate, secp_ctx: &Secp256k1<T>) -> Result<(), LightningError> {
                self.update_channel_intern(&msg.contents, Some(&msg), Some((&msg.signature, secp_ctx)))
        }
 
        /// For an already known (from announcement) channel, update info about one of the directions
        /// of the channel without verifying the associated signatures. Because we aren't given the
        /// associated signatures here we cannot relay the channel update to any of our peers.
-       pub fn update_channel_unsigned(&mut self, msg: &msgs::UnsignedChannelUpdate) -> Result<(), LightningError> {
+       pub fn update_channel_unsigned(&self, msg: &msgs::UnsignedChannelUpdate) -> Result<(), LightningError> {
                self.update_channel_intern(msg, None, None::<(&secp256k1::Signature, &Secp256k1<secp256k1::VerifyOnly>)>)
        }
 
-       fn update_channel_intern<T: secp256k1::Verification>(&mut self, msg: &msgs::UnsignedChannelUpdate, full_msg: Option<&msgs::ChannelUpdate>, sig_info: Option<(&secp256k1::Signature, &Secp256k1<T>)>) -> Result<(), LightningError> {
+       fn update_channel_intern<T: secp256k1::Verification>(&self, msg: &msgs::UnsignedChannelUpdate, full_msg: Option<&msgs::ChannelUpdate>, sig_info: Option<(&secp256k1::Signature, &Secp256k1<T>)>) -> Result<(), LightningError> {
                let dest_node_id;
                let chan_enabled = msg.flags & (1 << 1) != (1 << 1);
                let chan_was_enabled;
 
-               match self.channels.get_mut(&msg.short_channel_id) {
+               let mut channels = self.channels.write().unwrap();
+               match channels.get_mut(&msg.short_channel_id) {
                        None => return Err(LightningError{err: "Couldn't find channel for update".to_owned(), action: ErrorAction::IgnoreError}),
                        Some(channel) => {
                                if let OptionalField::Present(htlc_maximum_msat) = msg.htlc_maximum_msat {
@@ -1000,8 +1063,9 @@ impl NetworkGraph {
                        }
                }
 
+               let mut nodes = self.nodes.write().unwrap();
                if chan_enabled {
-                       let node = self.nodes.get_mut(&dest_node_id).unwrap();
+                       let node = nodes.get_mut(&dest_node_id).unwrap();
                        let mut base_msat = msg.fee_base_msat;
                        let mut proportional_millionths = msg.fee_proportional_millionths;
                        if let Some(fees) = node.lowest_inbound_channel_fees {
@@ -1013,11 +1077,11 @@ impl NetworkGraph {
                                proportional_millionths
                        });
                } else if chan_was_enabled {
-                       let node = self.nodes.get_mut(&dest_node_id).unwrap();
+                       let node = nodes.get_mut(&dest_node_id).unwrap();
                        let mut lowest_inbound_channel_fees = None;
 
                        for chan_id in node.channels.iter() {
-                               let chan = self.channels.get(chan_id).unwrap();
+                               let chan = channels.get(chan_id).unwrap();
                                let chan_info_opt;
                                if chan.node_one == dest_node_id {
                                        chan_info_opt = chan.two_to_one.as_ref();
@@ -1061,18 +1125,49 @@ impl NetworkGraph {
        }
 }
 
+impl ReadOnlyNetworkGraph<'_> {
+       /// Returns all known valid channels' short ids along with announced channel info.
+       ///
+       /// (C-not exported) because we have no mapping for `BTreeMap`s
+       pub fn channels(&self) -> &BTreeMap<u64, ChannelInfo> {
+               &*self.channels
+       }
+
+       /// Returns all known nodes' public keys along with announced node info.
+       ///
+       /// (C-not exported) because we have no mapping for `BTreeMap`s
+       pub fn nodes(&self) -> &BTreeMap<PublicKey, NodeInfo> {
+               &*self.nodes
+       }
+
+       /// Get network addresses by node id.
+       /// Returns None if the requested node is completely unknown,
+       /// or if node announcement for the node was never received.
+       ///
+       /// (C-not exported) as there is no practical way to track lifetimes of returned values.
+       pub fn get_addresses(&self, pubkey: &PublicKey) -> Option<&Vec<NetAddress>> {
+               if let Some(node) = self.nodes.get(pubkey) {
+                       if let Some(node_info) = node.announcement_info.as_ref() {
+                               return Some(&node_info.addresses)
+                       }
+               }
+               None
+       }
+}
+
 #[cfg(test)]
 mod tests {
        use chain;
+       use ln::PaymentHash;
        use ln::features::{ChannelFeatures, InitFeatures, NodeFeatures};
-       use routing::network_graph::{NetGraphMsgHandler, NetworkGraph, MAX_EXCESS_BYTES_FOR_RELAY};
+       use routing::network_graph::{NetGraphMsgHandler, NetworkGraph, NetworkUpdate, MAX_EXCESS_BYTES_FOR_RELAY};
        use ln::msgs::{Init, OptionalField, RoutingMessageHandler, UnsignedNodeAnnouncement, NodeAnnouncement,
-               UnsignedChannelAnnouncement, ChannelAnnouncement, UnsignedChannelUpdate, ChannelUpdate, HTLCFailChannelUpdate,
+               UnsignedChannelAnnouncement, ChannelAnnouncement, UnsignedChannelUpdate, ChannelUpdate, 
                ReplyChannelRange, ReplyShortChannelIdsEnd, QueryChannelRange, QueryShortChannelIds, MAX_VALUE_MSAT};
        use util::test_utils;
        use util::logger::Logger;
        use util::ser::{Readable, Writeable};
-       use util::events::{MessageSendEvent, MessageSendEventsProvider};
+       use util::events::{Event, EventHandler, MessageSendEvent, MessageSendEventsProvider};
        use util::scid_utils::scid_from_parts;
 
        use bitcoin::hashes::sha256d::Hash as Sha256dHash;
@@ -1096,7 +1191,8 @@ mod tests {
                let secp_ctx = Secp256k1::new();
                let logger = Arc::new(test_utils::TestLogger::new());
                let genesis_hash = genesis_block(Network::Testnet).header.block_hash();
-               let net_graph_msg_handler = NetGraphMsgHandler::new(genesis_hash, None, Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_hash);
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, Arc::clone(&logger));
                (secp_ctx, net_graph_msg_handler)
        }
 
@@ -1257,18 +1353,19 @@ mod tests {
                };
 
                // Test if the UTXO lookups were not supported
-               let mut net_graph_msg_handler = NetGraphMsgHandler::new(genesis_block(Network::Testnet).header.block_hash(), None, Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_block(Network::Testnet).header.block_hash());
+               let mut net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, Arc::clone(&logger));
                match net_graph_msg_handler.handle_channel_announcement(&valid_announcement) {
                        Ok(res) => assert!(res),
                        _ => panic!()
                };
 
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&unsigned_announcement.short_channel_id) {
+                       let network = &net_graph_msg_handler.network_graph;
+                       match network.read_only().channels().get(&unsigned_announcement.short_channel_id) {
                                None => panic!(),
                                Some(_) => ()
-                       }
+                       };
                }
 
                // If we receive announcement for the same channel (with UTXO lookups disabled),
@@ -1281,7 +1378,8 @@ mod tests {
                // Test if an associated transaction were not on-chain (or not confirmed).
                let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet));
                *chain_source.utxo_ret.lock().unwrap() = Err(chain::AccessError::UnknownTx);
-               net_graph_msg_handler = NetGraphMsgHandler::new(chain_source.clone().genesis_hash, Some(chain_source.clone()), Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_block(Network::Testnet).header.block_hash());
+               net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, Some(chain_source.clone()), Arc::clone(&logger));
                unsigned_announcement.short_channel_id += 1;
 
                msghash = hash_to_message!(&Sha256dHash::hash(&unsigned_announcement.encode()[..])[..]);
@@ -1316,11 +1414,11 @@ mod tests {
                };
 
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&unsigned_announcement.short_channel_id) {
+                       let network = &net_graph_msg_handler.network_graph;
+                       match network.read_only().channels().get(&unsigned_announcement.short_channel_id) {
                                None => panic!(),
                                Some(_) => ()
-                       }
+                       };
                }
 
                // If we receive announcement for the same channel (but TX is not confirmed),
@@ -1347,13 +1445,13 @@ mod tests {
                        _ => panic!()
                };
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&unsigned_announcement.short_channel_id) {
+                       let network = &net_graph_msg_handler.network_graph;
+                       match network.read_only().channels().get(&unsigned_announcement.short_channel_id) {
                                Some(channel_entry) => {
                                        assert_eq!(channel_entry.features, ChannelFeatures::empty());
                                },
                                _ => panic!()
-                       }
+                       };
                }
 
                // Don't relay valid channels with excess data
@@ -1405,7 +1503,8 @@ mod tests {
                let secp_ctx = Secp256k1::new();
                let logger: Arc<Logger> = Arc::new(test_utils::TestLogger::new());
                let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet));
-               let net_graph_msg_handler = NetGraphMsgHandler::new(genesis_block(Network::Testnet).header.block_hash(), Some(chain_source.clone()), Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_block(Network::Testnet).header.block_hash());
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, Some(chain_source.clone()), Arc::clone(&logger));
 
                let node_1_privkey = &SecretKey::from_slice(&[42; 32]).unwrap();
                let node_2_privkey = &SecretKey::from_slice(&[41; 32]).unwrap();
@@ -1477,14 +1576,14 @@ mod tests {
                };
 
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&short_channel_id) {
+                       let network = &net_graph_msg_handler.network_graph;
+                       match network.read_only().channels().get(&short_channel_id) {
                                None => panic!(),
                                Some(channel_info) => {
                                        assert_eq!(channel_info.one_to_two.as_ref().unwrap().cltv_expiry_delta, 144);
                                        assert!(channel_info.two_to_one.is_none());
                                }
-                       }
+                       };
                }
 
                unsigned_channel_update.timestamp += 100;
@@ -1569,8 +1668,14 @@ mod tests {
        }
 
        #[test]
-       fn handling_htlc_fail_channel_update() {
-               let (secp_ctx, net_graph_msg_handler) = create_net_graph_msg_handler();
+       fn handling_network_update() {
+               let logger = test_utils::TestLogger::new();
+               let chain_source = Arc::new(test_utils::TestChainSource::new(Network::Testnet));
+               let genesis_hash = genesis_block(Network::Testnet).header.block_hash();
+               let network_graph = NetworkGraph::new(genesis_hash);
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, Some(chain_source.clone()), &logger);
+               let secp_ctx = Secp256k1::new();
+
                let node_1_privkey = &SecretKey::from_slice(&[42; 32]).unwrap();
                let node_2_privkey = &SecretKey::from_slice(&[41; 32]).unwrap();
                let node_id_1 = PublicKey::from_secret_key(&secp_ctx, node_1_privkey);
@@ -1580,11 +1685,11 @@ mod tests {
 
                let short_channel_id = 0;
                let chain_hash = genesis_block(Network::Testnet).header.block_hash();
+               let network_graph = &net_graph_msg_handler.network_graph;
 
                {
                        // There is no nodes in the table at the beginning.
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       assert_eq!(network.get_nodes().len(), 0);
+                       assert_eq!(network_graph.read_only().nodes().len(), 0);
                }
 
                {
@@ -1608,10 +1713,9 @@ mod tests {
                                bitcoin_signature_2: secp_ctx.sign(&msghash, node_2_btckey),
                                contents: unsigned_announcement.clone(),
                        };
-                       match net_graph_msg_handler.handle_channel_announcement(&valid_channel_announcement) {
-                               Ok(_) => (),
-                               Err(_) => panic!()
-                       };
+                       let chain_source: Option<&test_utils::TestChainSource> = None;
+                       assert!(network_graph.update_channel_from_announcement(&valid_channel_announcement, &chain_source, &secp_ctx).is_ok());
+                       assert!(network_graph.read_only().channels().get(&short_channel_id).is_some());
 
                        let unsigned_channel_update = UnsignedChannelUpdate {
                                chain_hash,
@@ -1631,56 +1735,73 @@ mod tests {
                                contents: unsigned_channel_update.clone()
                        };
 
-                       match net_graph_msg_handler.handle_channel_update(&valid_channel_update) {
-                               Ok(res) => assert!(res),
-                               _ => panic!()
-                       };
+                       assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_none());
+
+                       net_graph_msg_handler.handle_event(&Event::PaymentPathFailed {
+                               payment_hash: PaymentHash([0; 32]),
+                               rejected_by_dest: false,
+                               all_paths_failed: true,
+                               path: vec![],
+                               network_update: Some(NetworkUpdate::ChannelUpdateMessage {
+                                       msg: valid_channel_update,
+                               }),
+                               error_code: None,
+                               error_data: None,
+                       });
+
+                       assert!(network_graph.read_only().channels().get(&short_channel_id).unwrap().one_to_two.is_some());
                }
 
                // Non-permanent closing just disables a channel
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&short_channel_id) {
+                       match network_graph.read_only().channels().get(&short_channel_id) {
                                None => panic!(),
                                Some(channel_info) => {
-                                       assert!(channel_info.one_to_two.is_some());
+                                       assert!(channel_info.one_to_two.as_ref().unwrap().enabled);
                                }
-                       }
-               }
-
-               let channel_close_msg = HTLCFailChannelUpdate::ChannelClosed {
-                       short_channel_id,
-                       is_permanent: false
-               };
+                       };
 
-               net_graph_msg_handler.handle_htlc_fail_channel_update(&channel_close_msg);
+                       net_graph_msg_handler.handle_event(&Event::PaymentPathFailed {
+                               payment_hash: PaymentHash([0; 32]),
+                               rejected_by_dest: false,
+                               all_paths_failed: true,
+                               path: vec![],
+                               network_update: Some(NetworkUpdate::ChannelClosed {
+                                       short_channel_id,
+                                       is_permanent: false,
+                               }),
+                               error_code: None,
+                               error_data: None,
+                       });
 
-               // Non-permanent closing just disables a channel
-               {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       match network.get_channels().get(&short_channel_id) {
+                       match network_graph.read_only().channels().get(&short_channel_id) {
                                None => panic!(),
                                Some(channel_info) => {
                                        assert!(!channel_info.one_to_two.as_ref().unwrap().enabled);
                                }
-                       }
+                       };
                }
 
-               let channel_close_msg = HTLCFailChannelUpdate::ChannelClosed {
-                       short_channel_id,
-                       is_permanent: true
-               };
-
-               net_graph_msg_handler.handle_htlc_fail_channel_update(&channel_close_msg);
-
                // Permanent closing deletes a channel
                {
-                       let network = net_graph_msg_handler.network_graph.read().unwrap();
-                       assert_eq!(network.get_channels().len(), 0);
+                       net_graph_msg_handler.handle_event(&Event::PaymentPathFailed {
+                               payment_hash: PaymentHash([0; 32]),
+                               rejected_by_dest: false,
+                               all_paths_failed: true,
+                               path: vec![],
+                               network_update: Some(NetworkUpdate::ChannelClosed {
+                                       short_channel_id,
+                                       is_permanent: true,
+                               }),
+                               error_code: None,
+                               error_data: None,
+                       });
+
+                       assert_eq!(network_graph.read_only().channels().len(), 0);
                        // Nodes are also deleted because there are no associated channels anymore
-                       assert_eq!(network.get_nodes().len(), 0);
+                       assert_eq!(network_graph.read_only().nodes().len(), 0);
                }
-               // TODO: Test HTLCFailChannelUpdate::NodeFailure, which is not implemented yet.
+               // TODO: Test NetworkUpdate::NodeFailure, which is not implemented yet.
        }
 
        #[test]
@@ -1993,10 +2114,10 @@ mod tests {
                        Err(_) => panic!()
                };
 
-               let network = net_graph_msg_handler.network_graph.write().unwrap();
+               let network = &net_graph_msg_handler.network_graph;
                let mut w = test_utils::TestVecWriter(Vec::new());
-               assert!(!network.get_nodes().is_empty());
-               assert!(!network.get_channels().is_empty());
+               assert!(!network.read_only().nodes().is_empty());
+               assert!(!network.read_only().channels().is_empty());
                network.write(&mut w).unwrap();
                assert!(<NetworkGraph>::read(&mut io::Cursor::new(&w.0)).unwrap() == *network);
        }
index 5030f6aaacbf9569a0ff5e2d26a20a31104c0c9b..325920b778151bafbfa98c8d610af77b7cb7ca41 100644 (file)
@@ -28,7 +28,7 @@ use core::cmp;
 use core::ops::Deref;
 
 /// A hop in a route
-#[derive(Clone, Hash, PartialEq, Eq)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct RouteHop {
        /// The node_id of the node at this hop.
        pub pubkey: PublicKey,
@@ -71,6 +71,27 @@ pub struct Route {
        pub paths: Vec<Vec<RouteHop>>,
 }
 
+impl Route {
+       /// Returns the total amount of fees paid on this [`Route`].
+       ///
+       /// This doesn't include any extra payment made to the recipient, which can happen in excess of
+       /// the amount passed to [`get_route`]'s `final_value_msat`.
+       pub fn get_total_fees(&self) -> u64 {
+               // Do not count last hop of each path since that's the full value of the payment
+               return self.paths.iter()
+                       .flat_map(|path| path.split_last().map(|(_, path_prefix)| path_prefix).unwrap_or(&[]))
+                       .map(|hop| &hop.fee_msat)
+                       .sum();
+       }
+
+       /// Returns the total amount paid on this [`Route`], excluding the fees.
+       pub fn get_total_amount(&self) -> u64 {
+               return self.paths.iter()
+                       .map(|path| path.split_last().map(|(hop, _)| hop.fee_msat).unwrap_or(0))
+                       .sum();
+       }
+}
+
 const SERIALIZATION_VERSION: u8 = 1;
 const MIN_SERIALIZATION_VERSION: u8 = 1;
 
@@ -443,6 +464,9 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        // to use as the A* heuristic beyond just the cost to get one node further than the current
        // one.
 
+       let network_graph = network.read_only();
+       let network_channels = network_graph.channels();
+       let network_nodes = network_graph.nodes();
        let dummy_directional_info = DummyDirectionalChannelInfo { // used for first_hops routes
                cltv_expiry_delta: 0,
                htlc_minimum_msat: 0,
@@ -458,7 +482,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        // work reliably.
        let allow_mpp = if let Some(features) = &payee_features {
                features.supports_basic_mpp()
-       } else if let Some(node) = network.get_nodes().get(&payee) {
+       } else if let Some(node) = network_nodes.get(&payee) {
                if let Some(node_info) = node.announcement_info.as_ref() {
                        node_info.features.supports_basic_mpp()
                } else { false }
@@ -492,7 +516,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
 
        // Map from node_id to information about the best current path to that node, including feerate
        // information.
-       let mut dist = HashMap::with_capacity(network.get_nodes().len());
+       let mut dist = HashMap::with_capacity(network_nodes.len());
 
        // During routing, if we ignore a path due to an htlc_minimum_msat limit, we set this,
        // indicating that we may wish to try again with a higher value, potentially paying to meet an
@@ -511,7 +535,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
        // This map allows paths to be aware of the channel use by other paths in the same call.
        // This would help to make a better path finding decisions and not "overbook" channels.
        // It is unaware of the directions (except for `outbound_capacity_msat` in `first_hops`).
-       let mut bookkeeped_channels_liquidity_available_msat = HashMap::with_capacity(network.get_nodes().len());
+       let mut bookkeeped_channels_liquidity_available_msat = HashMap::with_capacity(network_nodes.len());
 
        // Keeping track of how much value we already collected across other paths. Helps to decide:
        // - how much a new path should be transferring (upper bound);
@@ -629,7 +653,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                                        // as a way to reach the $dest_node_id.
                                                        let mut fee_base_msat = u32::max_value();
                                                        let mut fee_proportional_millionths = u32::max_value();
-                                                       if let Some(Some(fees)) = network.get_nodes().get(&$src_node_id).map(|node| node.lowest_inbound_channel_fees) {
+                                                       if let Some(Some(fees)) = network_nodes.get(&$src_node_id).map(|node| node.lowest_inbound_channel_fees) {
                                                                fee_base_msat = fees.base_msat;
                                                                fee_proportional_millionths = fees.proportional_millionths;
                                                        }
@@ -814,7 +838,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
 
                                if !features.requires_unknown_bits() {
                                        for chan_id in $node.channels.iter() {
-                                               let chan = network.get_channels().get(chan_id).unwrap();
+                                               let chan = network_channels.get(chan_id).unwrap();
                                                if !chan.features.requires_unknown_bits() {
                                                        if chan.node_one == *$node_id {
                                                                // ie $node is one, ie next hop in A* is two, via the two_to_one channel
@@ -862,7 +886,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
 
                // Add the payee as a target, so that the payee-to-payer
                // search algorithm knows what to start with.
-               match network.get_nodes().get(payee) {
+               match network_nodes.get(payee) {
                        // The payee is not in our network graph, so nothing to add here.
                        // There is still a chance of reaching them via last_hops though,
                        // so don't yet fail the payment here.
@@ -884,7 +908,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                // we have a direct channel to the first hop or the first hop is
                                // in the regular network graph.
                                first_hop_targets.get(&first_hop_in_route.src_node_id).is_some() ||
-                               network.get_nodes().get(&first_hop_in_route.src_node_id).is_some();
+                               network_nodes.get(&first_hop_in_route.src_node_id).is_some();
                        if have_hop_src_in_graph {
                                // We start building the path from reverse, i.e., from payee
                                // to the first RouteHintHop in the path.
@@ -991,7 +1015,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                                'path_walk: loop {
                                        if let Some(&(_, _, _, ref features)) = first_hop_targets.get(&ordered_hops.last().unwrap().0.pubkey) {
                                                ordered_hops.last_mut().unwrap().1 = features.clone();
-                                       } else if let Some(node) = network.get_nodes().get(&ordered_hops.last().unwrap().0.pubkey) {
+                                       } else if let Some(node) = network_nodes.get(&ordered_hops.last().unwrap().0.pubkey) {
                                                if let Some(node_info) = node.announcement_info.as_ref() {
                                                        ordered_hops.last_mut().unwrap().1 = node_info.features.clone();
                                                } else {
@@ -1093,7 +1117,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
                        // Otherwise, since the current target node is not us,
                        // keep "unrolling" the payment graph from payee to payer by
                        // finding a way to reach the current target from the payer side.
-                       match network.get_nodes().get(&pubkey) {
+                       match network_nodes.get(&pubkey) {
                                None => {},
                                Some(node) => {
                                        add_entries_to_cheapest_to_target_node!(node, &pubkey, lowest_fee_to_node, value_contribution_msat, path_htlc_minimum_msat);
@@ -1242,7 +1266,7 @@ pub fn get_route<L: Deref>(our_node_id: &PublicKey, network: &NetworkGraph, paye
 
 #[cfg(test)]
 mod tests {
-       use routing::router::{get_route, Route, RouteHint, RouteHintHop, RoutingFees};
+       use routing::router::{get_route, Route, RouteHint, RouteHintHop, RouteHop, RoutingFees};
        use routing::network_graph::{NetworkGraph, NetGraphMsgHandler};
        use chain::transaction::OutPoint;
        use ln::features::{ChannelFeatures, InitFeatures, InvoiceFeatures, NodeFeatures};
@@ -1293,8 +1317,10 @@ mod tests {
        }
 
        // Using the same keys for LN and BTC ids
-       fn add_channel(net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>, secp_ctx: &Secp256k1<All>, node_1_privkey: &SecretKey,
-          node_2_privkey: &SecretKey, features: ChannelFeatures, short_channel_id: u64) {
+       fn add_channel(
+               net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>,
+               secp_ctx: &Secp256k1<All>, node_1_privkey: &SecretKey, node_2_privkey: &SecretKey, features: ChannelFeatures, short_channel_id: u64
+       ) {
                let node_id_1 = PublicKey::from_secret_key(&secp_ctx, node_1_privkey);
                let node_id_2 = PublicKey::from_secret_key(&secp_ctx, node_2_privkey);
 
@@ -1323,7 +1349,10 @@ mod tests {
                };
        }
 
-       fn update_channel(net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>, secp_ctx: &Secp256k1<All>, node_privkey: &SecretKey, update: UnsignedChannelUpdate) {
+       fn update_channel(
+               net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>,
+               secp_ctx: &Secp256k1<All>, node_privkey: &SecretKey, update: UnsignedChannelUpdate
+       ) {
                let msghash = hash_to_message!(&Sha256dHash::hash(&update.encode()[..])[..]);
                let valid_channel_update = ChannelUpdate {
                        signature: secp_ctx.sign(&msghash, node_privkey),
@@ -1336,8 +1365,10 @@ mod tests {
                };
        }
 
-       fn add_or_update_node(net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>, secp_ctx: &Secp256k1<All>, node_privkey: &SecretKey,
-          features: NodeFeatures, timestamp: u32) {
+       fn add_or_update_node(
+               net_graph_msg_handler: &NetGraphMsgHandler<Arc<test_utils::TestChainSource>, Arc<test_utils::TestLogger>>,
+               secp_ctx: &Secp256k1<All>, node_privkey: &SecretKey, features: NodeFeatures, timestamp: u32
+       ) {
                let node_id = PublicKey::from_secret_key(&secp_ctx, node_privkey);
                let unsigned_announcement = UnsignedNodeAnnouncement {
                        features,
@@ -1393,7 +1424,8 @@ mod tests {
                let secp_ctx = Secp256k1::new();
                let logger = Arc::new(test_utils::TestLogger::new());
                let chain_monitor = Arc::new(test_utils::TestChainSource::new(Network::Testnet));
-               let net_graph_msg_handler = NetGraphMsgHandler::new(genesis_block(Network::Testnet).header.block_hash(), None, Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_block(Network::Testnet).header.block_hash());
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, Arc::clone(&logger));
                // Build network from our_id to node6:
                //
                //        -1(1)2-  node0  -1(3)2-
@@ -1701,11 +1733,11 @@ mod tests {
 
                // Simple route to 2 via 1
 
-               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 0, 42, Arc::clone(&logger)) {
+               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 0, 42, Arc::clone(&logger)) {
                        assert_eq!(err, "Cannot send a payment of 0 msat");
                } else { panic!(); }
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -1732,11 +1764,11 @@ mod tests {
 
                let our_chans = vec![get_channel_details(Some(2), our_id, InitFeatures::from_le_bytes(vec![0b11]), 100000)];
 
-               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)) {
+               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)) {
                        assert_eq!(err, "First hop cannot have our_node_id as a destination.");
                } else { panic!(); }
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
        }
 
@@ -1840,7 +1872,7 @@ mod tests {
                });
 
                // Not possible to send 199_999_999, because the minimum on channel=2 is 200_000_000.
-               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 199_999_999, 42, Arc::clone(&logger)) {
+               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 199_999_999, 42, Arc::clone(&logger)) {
                        assert_eq!(err, "Failed to find a path to the given destination");
                } else { panic!(); }
 
@@ -1859,7 +1891,7 @@ mod tests {
                });
 
                // A payment above the minimum should pass
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 199_999_999, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 199_999_999, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
        }
 
@@ -1936,7 +1968,7 @@ mod tests {
                        excess_data: Vec::new()
                });
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                        Some(InvoiceFeatures::known()), None, &Vec::new(), 60_000, 42, Arc::clone(&logger)).unwrap();
                // Overpay fees to hit htlc_minimum_msat.
                let overpaid_fees = route.paths[0][0].fee_msat + route.paths[1][0].fee_msat;
@@ -1982,7 +2014,7 @@ mod tests {
                        excess_data: Vec::new()
                });
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                        Some(InvoiceFeatures::known()), None, &Vec::new(), 60_000, 42, Arc::clone(&logger)).unwrap();
                // Fine to overpay for htlc_minimum_msat if it allows us to save fee.
                assert_eq!(route.paths.len(), 1);
@@ -1990,7 +2022,7 @@ mod tests {
                let fees = route.paths[0][0].fee_msat;
                assert_eq!(fees, 5_000);
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                        Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, 42, Arc::clone(&logger)).unwrap();
                // Not fine to overpay for htlc_minimum_msat if it requires paying more than fee on
                // the other channel.
@@ -2032,13 +2064,13 @@ mod tests {
                });
 
                // If all the channels require some features we don't understand, route should fail
-               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)) {
+               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)) {
                        assert_eq!(err, "Failed to find a path to the given destination");
                } else { panic!(); }
 
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()),  &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()),  &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[7]);
@@ -2068,13 +2100,13 @@ mod tests {
                add_or_update_node(&net_graph_msg_handler, &secp_ctx, &privkeys[7], unknown_features.clone(), 1);
 
                // If all nodes require some features we don't understand, route should fail
-               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)) {
+               if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)) {
                        assert_eq!(err, "Failed to find a path to the given destination");
                } else { panic!(); }
 
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[7]);
@@ -2102,7 +2134,7 @@ mod tests {
                let (_, our_id, _, nodes) = get_nodes(&secp_ctx);
 
                // Route to 1 via 2 and 3 because our channel to 1 is disabled
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[0], None, None, &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 3);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2128,7 +2160,7 @@ mod tests {
 
                // If we specify a channel to node7, that overrides our local channel view and that gets used
                let our_chans = vec![get_channel_details(Some(42), nodes[7].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[7]);
@@ -2247,12 +2279,12 @@ mod tests {
                let mut invalid_last_hops = last_hops_multi_private_channels(&nodes);
                invalid_last_hops.push(invalid_last_hop);
                {
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &invalid_last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)) {
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &invalid_last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Last hop cannot have a payee as a source.");
                        } else { panic!(); }
                }
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &last_hops_multi_private_channels(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &last_hops_multi_private_channels(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 5);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2324,7 +2356,7 @@ mod tests {
 
                // Test handling of an empty RouteHint passed in Invoice.
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &empty_last_hop(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &empty_last_hop(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 5);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2430,7 +2462,7 @@ mod tests {
                        excess_data: Vec::new()
                });
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &multi_hint_last_hops(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &multi_hint_last_hops(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 4);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2508,7 +2540,7 @@ mod tests {
                // This test shows that public routes can be present in the invoice
                // which would be handled in the same manner.
 
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &last_hops_with_public_channel(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &last_hops_with_public_channel(&nodes).iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 5);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2557,7 +2589,7 @@ mod tests {
                // Simple test with outbound channel to 4 to test that last_hops and first_hops connect
                let our_chans = vec![get_channel_details(Some(42), nodes[3].clone(), InitFeatures::from_le_bytes(vec![0b11]), 250_000_000)];
                let mut last_hops = last_hops(&nodes);
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, Some(&our_chans.iter().collect::<Vec<_>>()), &last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, Some(&our_chans.iter().collect::<Vec<_>>()), &last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 2);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[3]);
@@ -2577,7 +2609,7 @@ mod tests {
                last_hops[0].0[0].fees.base_msat = 1000;
 
                // Revert to via 6 as the fee on 8 goes up
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &last_hops.iter().collect::<Vec<_>>(), 100, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 4);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2611,7 +2643,7 @@ mod tests {
                assert_eq!(route.paths[0][3].channel_features.le_flags(), &Vec::<u8>::new()); // We can't learn any flags from invoices, sadly
 
                // ...but still use 8 for larger payments as 6 has a variable feerate
-               let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &last_hops.iter().collect::<Vec<_>>(), 2000, 42, Arc::clone(&logger)).unwrap();
+               let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &last_hops.iter().collect::<Vec<_>>(), 2000, 42, Arc::clone(&logger)).unwrap();
                assert_eq!(route.paths[0].len(), 5);
 
                assert_eq!(route.paths[0][0].pubkey, nodes[1]);
@@ -2785,7 +2817,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 250_000_001, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -2793,7 +2825,7 @@ mod tests {
 
                {
                        // Now, attempt to route an exact amount we have should be fine.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 250_000_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
@@ -2822,7 +2854,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 200_000_001, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -2830,7 +2862,7 @@ mod tests {
 
                {
                        // Now, attempt to route an exact amount we have should be fine.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), Some(&our_chans.iter().collect::<Vec<_>>()), &Vec::new(), 200_000_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
@@ -2870,7 +2902,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 15_001, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -2878,7 +2910,7 @@ mod tests {
 
                {
                        // Now, attempt to route an exact amount we have should be fine.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 15_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
@@ -2941,7 +2973,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 15_001, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -2949,7 +2981,7 @@ mod tests {
 
                {
                        // Now, attempt to route an exact amount we have should be fine.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 15_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
@@ -2974,7 +3006,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 10_001, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -2982,7 +3014,7 @@ mod tests {
 
                {
                        // Now, attempt to route an exact amount we have should be fine.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 10_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let path = route.paths.last().unwrap();
@@ -3082,7 +3114,7 @@ mod tests {
                });
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 60_000, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -3090,7 +3122,7 @@ mod tests {
 
                {
                        // Now, attempt to route 49 sats (just a bit below the capacity).
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 49_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
@@ -3104,7 +3136,7 @@ mod tests {
 
                {
                        // Attempt to route an exact amount is also fine
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 50_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
@@ -3149,7 +3181,7 @@ mod tests {
                });
 
                {
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 50_000, 42, Arc::clone(&logger)).unwrap();
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 50_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        let mut total_amount_paid_msat = 0;
                        for path in &route.paths {
@@ -3256,7 +3288,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(),
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph,
                                        &nodes[2], Some(InvoiceFeatures::known()), None, &Vec::new(), 300_000, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -3265,7 +3297,7 @@ mod tests {
                {
                        // Now, attempt to route 250 sats (just a bit below the capacity).
                        // Our algorithm should provide us with these 3 paths.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 250_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
@@ -3279,7 +3311,7 @@ mod tests {
 
                {
                        // Attempt to route an exact amount is also fine
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 290_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
@@ -3430,7 +3462,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 350_000, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -3439,7 +3471,7 @@ mod tests {
                {
                        // Now, attempt to route 300 sats (exact amount we can route).
                        // Our algorithm should provide us with these 3 paths, 100 sats each.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 300_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 3);
 
@@ -3596,7 +3628,7 @@ mod tests {
                {
                        // Now, attempt to route 180 sats.
                        // Our algorithm should provide us with these 2 paths.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 180_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 2);
 
@@ -3762,7 +3794,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 210_000, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -3770,7 +3802,7 @@ mod tests {
 
                {
                        // Now, attempt to route 200 sats (exact amount we can route).
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[3],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 200_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 2);
 
@@ -3780,6 +3812,7 @@ mod tests {
                                total_amount_paid_msat += path.last().unwrap().fee_msat;
                        }
                        assert_eq!(total_amount_paid_msat, 200_000);
+                       assert_eq!(route.get_total_fees(), 150_000);
                }
 
        }
@@ -3880,7 +3913,7 @@ mod tests {
 
                {
                        // Attempt to route more than available results in a failure.
-                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       if let Err(LightningError{err, action: ErrorAction::IgnoreError}) = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                        Some(InvoiceFeatures::known()), None, &Vec::new(), 150_000, 42, Arc::clone(&logger)) {
                                assert_eq!(err, "Failed to find a sufficient route to the given destination");
                        } else { panic!(); }
@@ -3889,7 +3922,7 @@ mod tests {
                {
                        // Now, attempt to route 125 sats (just a bit below the capacity of 3 channels).
                        // Our algorithm should provide us with these 3 paths.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 125_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 3);
                        let mut total_amount_paid_msat = 0;
@@ -3903,7 +3936,7 @@ mod tests {
 
                {
                        // Attempt to route without the last small cheap channel
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2],
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2],
                                Some(InvoiceFeatures::known()), None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 2);
                        let mut total_amount_paid_msat = 0;
@@ -3944,7 +3977,8 @@ mod tests {
                // "previous hop" being set to node 3, creating a loop in the path.
                let secp_ctx = Secp256k1::new();
                let logger = Arc::new(test_utils::TestLogger::new());
-               let net_graph_msg_handler = NetGraphMsgHandler::new(genesis_block(Network::Testnet).header.block_hash(), None, Arc::clone(&logger));
+               let network_graph = NetworkGraph::new(genesis_block(Network::Testnet).header.block_hash());
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, Arc::clone(&logger));
                let (our_privkey, our_id, privkeys, nodes) = get_nodes(&secp_ctx);
 
                add_channel(&net_graph_msg_handler, &secp_ctx, &our_privkey, &privkeys[1], ChannelFeatures::from_le_bytes(id_to_feature_flags(6)), 6);
@@ -4038,7 +4072,7 @@ mod tests {
 
                {
                        // Now ensure the route flows simply over nodes 1 and 4 to 6.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[6], None, None, &Vec::new(), 10_000, 42, Arc::clone(&logger)).unwrap();
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[6], None, None, &Vec::new(), 10_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        assert_eq!(route.paths[0].len(), 3);
 
@@ -4105,7 +4139,7 @@ mod tests {
                {
                        // Now, attempt to route 90 sats, which is exactly 90 sats at the last hop, plus the
                        // 200% fee charged channel 13 in the 1-to-2 direction.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], None, None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], None, None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        assert_eq!(route.paths[0].len(), 2);
 
@@ -4166,7 +4200,7 @@ mod tests {
                        // Now, attempt to route 90 sats, hitting the htlc_minimum on channel 4, but
                        // overshooting the htlc_maximum on channel 2. Thus, we should pick the (absurdly
                        // expensive) channels 12-13 path.
-                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2], Some(InvoiceFeatures::known()), None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
+                       let route = get_route(&our_id, &net_graph_msg_handler.network_graph, &nodes[2], Some(InvoiceFeatures::known()), None, &Vec::new(), 90_000, 42, Arc::clone(&logger)).unwrap();
                        assert_eq!(route.paths.len(), 1);
                        assert_eq!(route.paths[0].len(), 2);
 
@@ -4186,6 +4220,75 @@ mod tests {
                }
        }
 
+       #[test]
+       fn total_fees_single_path() {
+               let route = Route {
+                       paths: vec![vec![
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 100, cltv_expiry_delta: 0
+                               },
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0
+                               },
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("027f31ebc5462c1fdce1b737ecff52d37d75dea43ce11c74d25aa297165faa2007").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 225, cltv_expiry_delta: 0
+                               },
+                       ]],
+               };
+
+               assert_eq!(route.get_total_fees(), 250);
+               assert_eq!(route.get_total_amount(), 225);
+       }
+
+       #[test]
+       fn total_fees_multi_path() {
+               let route = Route {
+                       paths: vec![vec![
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 100, cltv_expiry_delta: 0
+                               },
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0
+                               },
+                       ],vec![
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("02eec7245d6b7d2ccb30380bfbe2a3648cd7a942653f5aa340edcea1f283686619").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 100, cltv_expiry_delta: 0
+                               },
+                               RouteHop {
+                                       pubkey: PublicKey::from_slice(&hex::decode("0324653eac434488002cc06bbfb7f10fe18991e35f9fe4302dbea6d2353dc0ab1c").unwrap()[..]).unwrap(),
+                                       channel_features: ChannelFeatures::empty(), node_features: NodeFeatures::empty(),
+                                       short_channel_id: 0, fee_msat: 150, cltv_expiry_delta: 0
+                               },
+                       ]],
+               };
+
+               assert_eq!(route.get_total_fees(), 200);
+               assert_eq!(route.get_total_amount(), 300);
+       }
+
+       #[test]
+       fn total_empty_route_no_panic() {
+               // In an earlier version of `Route::get_total_fees` and `Route::get_total_amount`, they
+               // would both panic if the route was completely empty. We test to ensure they return 0
+               // here, even though its somewhat nonsensical as a route.
+               let route = Route { paths: Vec::new() };
+
+               assert_eq!(route.get_total_fees(), 0);
+               assert_eq!(route.get_total_amount(), 0);
+       }
+
        #[cfg(not(feature = "no-std"))]
        pub(super) fn random_init_seed() -> u64 {
                // Because the default HashMap in std pulls OS randomness, we can use it as a (bad) RNG.
@@ -4211,12 +4314,13 @@ mod tests {
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
                let mut seed = random_init_seed() as usize;
+               let nodes = graph.read_only().nodes().clone();
                'load_endpoints: for _ in 0..10 {
                        loop {
                                seed = seed.overflowing_mul(0xdeadbeef).0;
-                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let src = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                seed = seed.overflowing_mul(0xdeadbeef).0;
-                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let dst = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                let amt = seed as u64 % 200_000_000;
                                if get_route(src, &graph, dst, None, None, &[], amt, 42, &test_utils::TestLogger::new()).is_ok() {
                                        continue 'load_endpoints;
@@ -4239,12 +4343,13 @@ mod tests {
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
                let mut seed = random_init_seed() as usize;
+               let nodes = graph.read_only().nodes().clone();
                'load_endpoints: for _ in 0..10 {
                        loop {
                                seed = seed.overflowing_mul(0xdeadbeef).0;
-                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let src = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                seed = seed.overflowing_mul(0xdeadbeef).0;
-                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let dst = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                let amt = seed as u64 % 200_000_000;
                                if get_route(src, &graph, dst, Some(InvoiceFeatures::known()), None, &[], amt, 42, &test_utils::TestLogger::new()).is_ok() {
                                        continue 'load_endpoints;
@@ -4297,6 +4402,7 @@ mod benches {
        fn generate_routes(bench: &mut Bencher) {
                let mut d = test_utils::get_route_file().unwrap();
                let graph = NetworkGraph::read(&mut d).unwrap();
+               let nodes = graph.read_only().nodes().clone();
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
                let mut path_endpoints = Vec::new();
@@ -4304,9 +4410,9 @@ mod benches {
                'load_endpoints: for _ in 0..100 {
                        loop {
                                seed *= 0xdeadbeef;
-                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let src = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                seed *= 0xdeadbeef;
-                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let dst = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                let amt = seed as u64 % 1_000_000;
                                if get_route(src, &graph, dst, None, None, &[], amt, 42, &DummyLogger{}).is_ok() {
                                        path_endpoints.push((src, dst, amt));
@@ -4328,6 +4434,7 @@ mod benches {
        fn generate_mpp_routes(bench: &mut Bencher) {
                let mut d = test_utils::get_route_file().unwrap();
                let graph = NetworkGraph::read(&mut d).unwrap();
+               let nodes = graph.read_only().nodes().clone();
 
                // First, get 100 (source, destination) pairs for which route-getting actually succeeds...
                let mut path_endpoints = Vec::new();
@@ -4335,9 +4442,9 @@ mod benches {
                'load_endpoints: for _ in 0..100 {
                        loop {
                                seed *= 0xdeadbeef;
-                               let src = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let src = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                seed *= 0xdeadbeef;
-                               let dst = graph.get_nodes().keys().skip(seed % graph.get_nodes().len()).next().unwrap();
+                               let dst = nodes.keys().skip(seed % nodes.len()).next().unwrap();
                                let amt = seed as u64 % 1_000_000;
                                if get_route(src, &graph, dst, Some(InvoiceFeatures::known()), None, &[], amt, 42, &DummyLogger{}).is_ok() {
                                        path_endpoints.push((src, dst, amt));
index b7dfe767057836cdd59dd61c5bae32d88c362c6a..b905665c98918bbb8f1fd9306174badb51752724 100644 (file)
@@ -7,7 +7,7 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-use ln::chan_utils::{HTLCOutputInCommitment, ChannelPublicKeys, HolderCommitmentTransaction, CommitmentTransaction, ChannelTransactionParameters, TrustedCommitmentTransaction};
+use ln::chan_utils::{HTLCOutputInCommitment, ChannelPublicKeys, HolderCommitmentTransaction, CommitmentTransaction, ChannelTransactionParameters, TrustedCommitmentTransaction, ClosingTransaction};
 use ln::{chan_utils, msgs};
 use chain::keysinterface::{Sign, InMemorySigner, BaseSign};
 
@@ -182,7 +182,9 @@ impl BaseSign for EnforcingSigner {
                Ok(self.inner.sign_counterparty_htlc_transaction(htlc_tx, input, amount, per_commitment_point, htlc, secp_ctx).unwrap())
        }
 
-       fn sign_closing_transaction(&self, closing_tx: &Transaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
+       fn sign_closing_transaction(&self, closing_tx: &ClosingTransaction, secp_ctx: &Secp256k1<secp256k1::All>) -> Result<Signature, ()> {
+               closing_tx.verify(self.inner.funding_outpoint().into_bitcoin_outpoint())
+                       .expect("derived different closing transaction");
                Ok(self.inner.sign_closing_transaction(closing_tx, secp_ctx).unwrap())
        }
 
index 1780483cb8b9e01bcab8d5f0c18e812522d27a7f..c8cedeb14cd9ac8cfaebb070733dcc130d5462ba 100644 (file)
 //! future, as well as generate and broadcast funding transactions handle payment preimages and a
 //! few other things.
 
+use chain::keysinterface::SpendableOutputDescriptor;
 use ln::msgs;
+use ln::msgs::DecodeError;
 use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-use chain::keysinterface::SpendableOutputDescriptor;
-use util::ser::{Writeable, Writer, MaybeReadable, Readable, VecReadWrapper, VecWriteWrapper};
+use routing::network_graph::NetworkUpdate;
+use util::ser::{BigSize, FixedLengthReader, Writeable, Writer, MaybeReadable, Readable, VecReadWrapper, VecWriteWrapper};
+use routing::router::RouteHop;
 
 use bitcoin::blockdata::script::Script;
 
@@ -67,6 +70,60 @@ pub enum PaymentPurpose {
        SpontaneousPayment(PaymentPreimage),
 }
 
+#[derive(Clone, Debug, PartialEq)]
+/// The reason the channel was closed. See individual variants more details.
+pub enum ClosureReason {
+       /// Closure generated from receiving a peer error message.
+       ///
+       /// Our counterparty may have broadcasted their latest commitment state, and we have
+       /// as well.
+       CounterpartyForceClosed {
+               /// The error which the peer sent us.
+               ///
+               /// The string should be sanitized before it is used (e.g emitted to logs
+               /// or printed to stdout). Otherwise, a well crafted error message may exploit
+               /// a security vulnerability in the terminal emulator or the logging subsystem.
+               peer_msg: String,
+       },
+       /// Closure generated from [`ChannelManager::force_close_channel`], called by the user.
+       ///
+       /// [`ChannelManager::force_close_channel`]: crate::ln::channelmanager::ChannelManager::force_close_channel.
+       HolderForceClosed,
+       /// The channel was closed after negotiating a cooperative close and we've now broadcasted
+       /// the cooperative close transaction. Note the shutdown may have been initiated by us.
+       //TODO: split between CounterpartyInitiated/LocallyInitiated
+       CooperativeClosure,
+       /// A commitment transaction was confirmed on chain, closing the channel. Most likely this
+       /// commitment transaction came from our counterparty, but it may also have come from
+       /// a copy of our own `ChannelMonitor`.
+       CommitmentTxConfirmed,
+       /// Closure generated from processing an event, likely a HTLC forward/relay/reception.
+       ProcessingError {
+               /// A developer-readable error message which we generated.
+               err: String,
+       },
+       /// The `PeerManager` informed us that we've disconnected from the peer. We close channels
+       /// if the `PeerManager` informed us that it is unlikely we'll be able to connect to the
+       /// peer again in the future or if the peer disconnected before we finished negotiating
+       /// the channel open. The first case may be caused by incompatible features which our
+       /// counterparty, or we, require.
+       //TODO: split between PeerUnconnectable/PeerDisconnected ?
+       DisconnectedPeer,
+       /// Closure generated from `ChannelManager::read` if the ChannelMonitor is newer than
+       /// the ChannelManager deserialized.
+       OutdatedChannelManager
+}
+
+impl_writeable_tlv_based_enum_upgradable!(ClosureReason,
+       (0, CounterpartyForceClosed) => { (1, peer_msg, required) },
+       (2, HolderForceClosed) => {},
+       (6, CommitmentTxConfirmed) => {},
+       (4, CooperativeClosure) => {},
+       (8, ProcessingError) => { (1, err, required) },
+       (10, DisconnectedPeer) => {},
+       (12, OutdatedChannelManager) => {},
+);
+
 /// An Event which you should probably take some action in response to.
 ///
 /// Note that while Writeable and Readable are implemented for Event, you probably shouldn't use
@@ -111,8 +168,11 @@ pub enum Event {
                /// payment is to pay an invoice or to send a spontaneous payment.
                purpose: PaymentPurpose,
        },
-       /// Indicates an outbound payment we made succeeded (ie it made it all the way to its target
+       /// Indicates an outbound payment we made succeeded (i.e. it made it all the way to its target
        /// and we got back the payment preimage for it).
+       ///
+       /// Note for MPP payments: in rare cases, this event may be preceded by a `PaymentPathFailed`
+       /// event. In this situation, you SHOULD treat this payment as having succeeded.
        PaymentSent {
                /// The preimage to the hash given to ChannelManager::send_payment.
                /// Note that this serves as a payment receipt, if you wish to have such a thing, you must
@@ -121,13 +181,28 @@ pub enum Event {
        },
        /// Indicates an outbound payment we made failed. Probably some intermediary node dropped
        /// something. You may wish to retry with a different route.
-       PaymentFailed {
+       PaymentPathFailed {
                /// The hash which was given to ChannelManager::send_payment.
                payment_hash: PaymentHash,
                /// Indicates the payment was rejected for some reason by the recipient. This implies that
                /// the payment has failed, not just the route in question. If this is not set, you may
                /// retry the payment via a different route.
                rejected_by_dest: bool,
+               /// Any failure information conveyed via the Onion return packet by a node along the failed
+               /// payment route.
+               ///
+               /// Should be applied to the [`NetworkGraph`] so that routing decisions can take into
+               /// account the update. [`NetGraphMsgHandler`] is capable of doing this.
+               ///
+               /// [`NetworkGraph`]: crate::routing::network_graph::NetworkGraph
+               /// [`NetGraphMsgHandler`]: crate::routing::network_graph::NetGraphMsgHandler
+               network_update: Option<NetworkUpdate>,
+               /// For both single-path and multi-path payments, this is set if all paths of the payment have
+               /// failed. This will be set to false if (1) this is an MPP payment and (2) other parts of the
+               /// larger MPP payment were still in flight when this event was generated.
+               all_paths_failed: bool,
+               /// The payment path that failed.
+               path: Vec<RouteHop>,
 #[cfg(test)]
                error_code: Option<u16>,
 #[cfg(test)]
@@ -172,6 +247,14 @@ pub enum Event {
                /// transaction.
                claim_from_onchain_tx: bool,
        },
+       /// Used to indicate that a channel with the given `channel_id` is in the process of closure.
+       ChannelClosed  {
+               /// The channel_id of the channel which has been closed. Note that on-chain transactions
+               /// resolving the channel are likely still awaiting confirmation.
+               channel_id: [u8; 32],
+               /// The reason the channel was closed.
+               reason: ClosureReason
+       }
 }
 
 impl Writeable for Event {
@@ -211,7 +294,8 @@ impl Writeable for Event {
                                        (0, payment_preimage, required),
                                });
                        },
-                       &Event::PaymentFailed { ref payment_hash, ref rejected_by_dest,
+                       &Event::PaymentPathFailed { ref payment_hash, ref rejected_by_dest, ref network_update,
+                                                   ref all_paths_failed, ref path,
                                #[cfg(test)]
                                ref error_code,
                                #[cfg(test)]
@@ -224,14 +308,16 @@ impl Writeable for Event {
                                error_data.write(writer)?;
                                write_tlv_fields!(writer, {
                                        (0, payment_hash, required),
+                                       (1, network_update, option),
                                        (2, rejected_by_dest, required),
+                                       (3, all_paths_failed, required),
+                                       (5, path, vec_type),
                                });
                        },
                        &Event::PendingHTLCsForwardable { time_forwardable: _ } => {
                                4u8.write(writer)?;
-                               write_tlv_fields!(writer, {});
-                               // We don't write the time_fordwardable out at all, as we presume when the user
-                               // deserializes us at least that much time has elapsed.
+                               // Note that we now ignore these on the read end as we'll re-generate them in
+                               // ChannelManager, we write them here only for backwards compatibility.
                        },
                        &Event::SpendableOutputs { ref outputs } => {
                                5u8.write(writer)?;
@@ -246,6 +332,16 @@ impl Writeable for Event {
                                        (2, claim_from_onchain_tx, required),
                                });
                        },
+                       &Event::ChannelClosed { ref channel_id, ref reason } => {
+                               9u8.write(writer)?;
+                               write_tlv_fields!(writer, {
+                                       (0, channel_id, required),
+                                       (2, reason, required)
+                               });
+                       },
+                       // Note that, going forward, all new events must only write data inside of
+                       // `write_tlv_fields`. Versions 0.0.101+ will ignore odd-numbered events that write
+                       // data via `write_tlv_fields`.
                }
                Ok(())
        }
@@ -253,6 +349,8 @@ impl Writeable for Event {
 impl MaybeReadable for Event {
        fn read<R: io::Read>(reader: &mut R) -> Result<Option<Self>, msgs::DecodeError> {
                match Readable::read(reader)? {
+                       // Note that we do not write a length-prefixed TLV for FundingGenerationReady events,
+                       // unlike all other events, thus we return immediately here.
                        0u8 => Ok(None),
                        1u8 => {
                                let f = || {
@@ -307,13 +405,22 @@ impl MaybeReadable for Event {
                                        let error_data = Readable::read(reader)?;
                                        let mut payment_hash = PaymentHash([0; 32]);
                                        let mut rejected_by_dest = false;
+                                       let mut network_update = None;
+                                       let mut all_paths_failed = Some(true);
+                                       let mut path: Option<Vec<RouteHop>> = Some(vec![]);
                                        read_tlv_fields!(reader, {
                                                (0, payment_hash, required),
+                                               (1, network_update, ignorable),
                                                (2, rejected_by_dest, required),
+                                               (3, all_paths_failed, option),
+                                               (5, path, vec_type),
                                        });
-                                       Ok(Some(Event::PaymentFailed {
+                                       Ok(Some(Event::PaymentPathFailed {
                                                payment_hash,
                                                rejected_by_dest,
+                                               network_update,
+                                               all_paths_failed: all_paths_failed.unwrap(),
+                                               path: path.unwrap(),
                                                #[cfg(test)]
                                                error_code,
                                                #[cfg(test)]
@@ -322,15 +429,7 @@ impl MaybeReadable for Event {
                                };
                                f()
                        },
-                       4u8 => {
-                               let f = || {
-                                       read_tlv_fields!(reader, {});
-                                       Ok(Some(Event::PendingHTLCsForwardable {
-                                               time_forwardable: Duration::from_secs(0)
-                                       }))
-                               };
-                               f()
-                       },
+                       4u8 => Ok(None),
                        5u8 => {
                                let f = || {
                                        let mut outputs = VecReadWrapper(Vec::new());
@@ -353,8 +452,30 @@ impl MaybeReadable for Event {
                                };
                                f()
                        },
+                       9u8 => {
+                               let mut channel_id = [0; 32];
+                               let mut reason = None;
+                               read_tlv_fields!(reader, {
+                                       (0, channel_id, required),
+                                       (2, reason, ignorable),
+                               });
+                               if reason.is_none() { return Ok(None); }
+                               Ok(Some(Event::ChannelClosed { channel_id, reason: reason.unwrap() }))
+                       },
                        // Versions prior to 0.0.100 did not ignore odd types, instead returning InvalidValue.
-                       x if x % 2 == 1 => Ok(None),
+                       // Version 0.0.100 failed to properly ignore odd types, possibly resulting in corrupt
+                       // reads.
+                       x if x % 2 == 1 => {
+                               // If the event is of unknown type, assume it was written with `write_tlv_fields`,
+                               // which prefixes the whole thing with a length BigSize. Because the event is
+                               // odd-type unknown, we should treat it as `Ok(None)` even if it has some TLV
+                               // fields that are even. Thus, we avoid using `read_tlv_fields` and simply read
+                               // exactly the number of bytes specified, ignoring them entirely.
+                               let tlv_len: BigSize = Readable::read(reader)?;
+                               FixedLengthReader::new(reader, tlv_len.0)
+                                       .eat_remaining().map_err(|_| msgs::DecodeError::ShortRead)?;
+                               Ok(None)
+                       },
                        _ => Err(msgs::DecodeError::InvalidValue)
                }
        }
@@ -485,12 +606,6 @@ pub enum MessageSendEvent {
                /// The action which should be taken.
                action: msgs::ErrorAction
        },
-       /// When a payment fails we may receive updates back from the hop where it failed. In such
-       /// cases this event is generated so that we can inform the network graph of this information.
-       PaymentFailureNetworkUpdate {
-               /// The channel/node update which should be sent to NetGraphMsgHandler
-               update: msgs::HTLCFailChannelUpdate,
-       },
        /// Query a peer for channels with funding transaction UTXOs in a block range.
        SendChannelRangeQuery {
                /// The node_id of this message recipient
@@ -561,11 +676,11 @@ pub trait EventHandler {
        /// Handles the given [`Event`].
        ///
        /// See [`EventsProvider`] for details that must be considered when implementing this method.
-       fn handle_event(&self, event: Event);
+       fn handle_event(&self, event: &Event);
 }
 
-impl<F> EventHandler for F where F: Fn(Event) {
-       fn handle_event(&self, event: Event) {
+impl<F> EventHandler for F where F: Fn(&Event) {
+       fn handle_event(&self, event: &Event) {
                self(event)
        }
 }
index 9644411bc147310b51a506e9a6634adcdb98bfc3..d7849d77ae0b970796208ca7a52f726671047c26 100644 (file)
@@ -142,7 +142,7 @@ impl<'a> core::fmt::Display for DebugSpendable<'a> {
                                write!(f, "DelayedPaymentOutput {}:{} marked for spending", descriptor.outpoint.txid, descriptor.outpoint.index)?;
                        }
                        &SpendableOutputDescriptor::StaticPaymentOutput(ref descriptor) => {
-                               write!(f, "DynamicOutputP2WPKH {}:{} marked for spending", descriptor.outpoint.txid, descriptor.outpoint.index)?;
+                               write!(f, "StaticPaymentOutput {}:{} marked for spending", descriptor.outpoint.txid, descriptor.outpoint.index)?;
                        }
                }
                Ok(())
index 0b5036bd749424dd4529cb23481d5e01236d6974..47bdf04f8750ac8d2562f032c7232d997f36c3b4 100644 (file)
@@ -35,18 +35,13 @@ use util::byte_utils::{be48_to_array, slice_to_be48};
 /// serialization buffer size
 pub const MAX_BUF_SIZE: usize = 64 * 1024;
 
-/// A trait that is similar to std::io::Write but has one extra function which can be used to size
-/// buffers being written into.
-/// An impl is provided for any type that also impls std::io::Write which simply ignores size
-/// hints.
+/// A simplified version of std::io::Write that exists largely for backwards compatibility.
+/// An impl is provided for any type that also impls std::io::Write.
 ///
 /// (C-not exported) as we only export serialization to/from byte arrays instead
 pub trait Writer {
        /// Writes the given buf out. See std::io::Write::write_all for more
        fn write_all(&mut self, buf: &[u8]) -> Result<(), io::Error>;
-       /// Hints that data of the given size is about the be written. This may not always be called
-       /// prior to data being written and may be safely ignored.
-       fn size_hint(&mut self, size: usize);
 }
 
 impl<W: Write> Writer for W {
@@ -54,8 +49,6 @@ impl<W: Write> Writer for W {
        fn write_all(&mut self, buf: &[u8]) -> Result<(), io::Error> {
                <Self as io::Write>::write_all(self, buf)
        }
-       #[inline]
-       fn size_hint(&mut self, _size: usize) { }
 }
 
 pub(crate) struct WriterWriteAdaptor<'a, W: Writer + 'a>(pub &'a mut W);
@@ -82,10 +75,6 @@ impl Writer for VecWriter {
                self.0.extend_from_slice(buf);
                Ok(())
        }
-       #[inline]
-       fn size_hint(&mut self, size: usize) {
-               self.0.reserve_exact(size);
-       }
 }
 
 /// Writer that only tracks the amount of data written - useful if you need to calculate the length
@@ -97,8 +86,6 @@ impl Writer for LengthCalculatingWriter {
                self.0 += buf.len();
                Ok(())
        }
-       #[inline]
-       fn size_hint(&mut self, _size: usize) {}
 }
 
 /// Essentially std::io::Take but a bit simpler and with a method to walk the underlying stream
@@ -528,6 +515,36 @@ impl<K, V> Readable for HashMap<K, V>
        }
 }
 
+// HashSet
+impl<T> Writeable for HashSet<T>
+where T: Writeable + Eq + Hash
+{
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               (self.len() as u16).write(w)?;
+               for item in self.iter() {
+                       item.write(w)?;
+               }
+               Ok(())
+       }
+}
+
+impl<T> Readable for HashSet<T>
+where T: Readable + Eq + Hash
+{
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let len: u16 = Readable::read(r)?;
+               let mut ret = HashSet::with_capacity(len as usize);
+               for _ in 0..len {
+                       if !ret.insert(T::read(r)?) {
+                               return Err(DecodeError::InvalidValue)
+                       }
+               }
+               Ok(ret)
+       }
+}
+
 // Vectors
 impl Writeable for Vec<u8> {
        #[inline]
@@ -862,8 +879,29 @@ impl<A: Writeable, B: Writeable, C: Writeable> Writeable for (A, B, C) {
        }
 }
 
+impl Writeable for () {
+       fn write<W: Writer>(&self, _: &mut W) -> Result<(), io::Error> {
+               Ok(())
+       }
+}
 impl Readable for () {
        fn read<R: Read>(_r: &mut R) -> Result<Self, DecodeError> {
                Ok(())
        }
 }
+
+impl Writeable for String {
+       #[inline]
+       fn write<W: Writer>(&self, w: &mut W) -> Result<(), io::Error> {
+               (self.len() as u16).write(w)?;
+               w.write_all(self.as_bytes())
+       }
+}
+impl Readable for String {
+       #[inline]
+       fn read<R: Read>(r: &mut R) -> Result<Self, DecodeError> {
+               let v: Vec<u8> = Readable::read(r)?;
+               let ret = String::from_utf8(v).map_err(|_| DecodeError::InvalidValue)?;
+               Ok(ret)
+       }
+}
index 5178732c7457e93745d39319fa2f2ecee99a80dd..b80ca537bbf03bd86d07912687d32ce81cd11f81 100644 (file)
@@ -154,7 +154,8 @@ macro_rules! decode_tlv {
                $field = ser::Readable::read(&mut $reader)?;
        }};
        ($reader: expr, $field: ident, vec_type) => {{
-               $field = Some(ser::Readable::read(&mut $reader)?);
+               let f: ::util::ser::VecReadWrapper<_> = ser::Readable::read(&mut $reader)?;
+               $field = Some(f.0);
        }};
        ($reader: expr, $field: ident, option) => {{
                $field = Some(ser::Readable::read(&mut $reader)?);
@@ -230,103 +231,52 @@ macro_rules! decode_tlv_stream {
        } }
 }
 
-macro_rules! impl_writeable {
-       ($st:ident, $len: expr, {$($field:ident),*}) => {
+macro_rules! impl_writeable_msg {
+       ($st:ident, {$($field:ident),* $(,)*}, {$(($type: expr, $tlvfield: ident, $fieldty: tt)),* $(,)*}) => {
                impl ::util::ser::Writeable for $st {
                        fn write<W: ::util::ser::Writer>(&self, w: &mut W) -> Result<(), $crate::io::Error> {
-                               if $len != 0 {
-                                       w.size_hint($len);
-                               }
-                               #[cfg(any(test, feature = "fuzztarget"))]
-                               {
-                                       // In tests, assert that the hard-coded length matches the actual one
-                                       if $len != 0 {
-                                               let mut len_calc = ::util::ser::LengthCalculatingWriter(0);
-                                               $( self.$field.write(&mut len_calc).expect("No in-memory data may fail to serialize"); )*
-                                               assert_eq!(len_calc.0, $len);
-                                               assert_eq!(self.serialized_length(), $len);
-                                       }
-                               }
                                $( self.$field.write(w)?; )*
+                               encode_tlv_stream!(w, {$(($type, self.$tlvfield, $fieldty)),*});
                                Ok(())
                        }
-
-                       #[inline]
-                       fn serialized_length(&self) -> usize {
-                               if $len == 0 || cfg!(any(test, feature = "fuzztarget")) {
-                                       let mut len_calc = 0;
-                                       $( len_calc += self.$field.serialized_length(); )*
-                                       if $len != 0 {
-                                               // In tests, assert that the hard-coded length matches the actual one
-                                               assert_eq!(len_calc, $len);
-                                       } else {
-                                               return len_calc;
-                                       }
-                               }
-                               $len
-                       }
                }
-
                impl ::util::ser::Readable for $st {
                        fn read<R: $crate::io::Read>(r: &mut R) -> Result<Self, ::ln::msgs::DecodeError> {
+                               $(let $field = ::util::ser::Readable::read(r)?;)*
+                               $(init_tlv_field_var!($tlvfield, $fieldty);)*
+                               decode_tlv_stream!(r, {$(($type, $tlvfield, $fieldty)),*});
                                Ok(Self {
-                                       $($field: ::util::ser::Readable::read(r)?),*
+                                       $($field),*,
+                                       $($tlvfield),*
                                })
                        }
                }
        }
 }
-macro_rules! impl_writeable_len_match {
-       ($struct: ident, $cmp: tt, ($calc_len: expr), {$({$match: pat, $length: expr}),*}, {$($field:ident),*}) => {
-               impl Writeable for $struct {
-                       fn write<W: Writer>(&self, w: &mut W) -> Result<(), $crate::io::Error> {
-                               let len = match *self {
-                                       $($match => $length,)*
-                               };
-                               w.size_hint(len);
-                               #[cfg(any(test, feature = "fuzztarget"))]
-                               {
-                                       // In tests, assert that the hard-coded length matches the actual one
-                                       let mut len_calc = ::util::ser::LengthCalculatingWriter(0);
-                                       $( self.$field.write(&mut len_calc).expect("No in-memory data may fail to serialize"); )*
-                                       assert!(len_calc.0 $cmp len);
-                                       assert_eq!(len_calc.0, self.serialized_length());
-                               }
+
+macro_rules! impl_writeable {
+       ($st:ident, {$($field:ident),*}) => {
+               impl ::util::ser::Writeable for $st {
+                       fn write<W: ::util::ser::Writer>(&self, w: &mut W) -> Result<(), $crate::io::Error> {
                                $( self.$field.write(w)?; )*
                                Ok(())
                        }
 
                        #[inline]
                        fn serialized_length(&self) -> usize {
-                               if $calc_len || cfg!(any(test, feature = "fuzztarget")) {
-                                       let mut len_calc = 0;
-                                       $( len_calc += self.$field.serialized_length(); )*
-                                       if !$calc_len {
-                                               assert_eq!(len_calc, match *self {
-                                                       $($match => $length,)*
-                                               });
-                                       }
-                                       return len_calc
-                               }
-                               match *self {
-                                       $($match => $length,)*
-                               }
+                               let mut len_calc = 0;
+                               $( len_calc += self.$field.serialized_length(); )*
+                               return len_calc;
                        }
                }
 
-               impl ::util::ser::Readable for $struct {
-                       fn read<R: $crate::io::Read>(r: &mut R) -> Result<Self, DecodeError> {
+               impl ::util::ser::Readable for $st {
+                       fn read<R: $crate::io::Read>(r: &mut R) -> Result<Self, ::ln::msgs::DecodeError> {
                                Ok(Self {
-                                       $($field: Readable::read(r)?),*
+                                       $($field: ::util::ser::Readable::read(r)?),*
                                })
                        }
                }
-       };
-       ($struct: ident, $cmp: tt, {$({$match: pat, $length: expr}),*}, {$($field:ident),*}) => {
-               impl_writeable_len_match!($struct, $cmp, (true), { $({ $match, $length }),* }, { $($field),* });
-       };
-       ($struct: ident, {$({$match: pat, $length: expr}),*}, {$($field:ident),*}) => {
-               impl_writeable_len_match!($struct, ==, (false), { $({ $match, $length }),* }, { $($field),* });
        }
 }
 
@@ -399,7 +349,7 @@ macro_rules! init_tlv_based_struct_field {
                $field.0.unwrap()
        };
        ($field: ident, vec_type) => {
-               $field.unwrap().0
+               $field.unwrap()
        };
 }
 
@@ -411,7 +361,7 @@ macro_rules! init_tlv_field_var {
                let mut $field = ::util::ser::OptionDeserWrapper(None);
        };
        ($field: ident, vec_type) => {
-               let mut $field = Some(::util::ser::VecReadWrapper(Vec::new()));
+               let mut $field = Some(Vec::new());
        };
        ($field: ident, option) => {
                let mut $field = None;
index 64b88acb0081487495f997ac2706f9783dd62d9c..aa3fcb550f62c83b45e2eb717bbfa7e83f946a2a 100644 (file)
@@ -52,9 +52,6 @@ impl Writer for TestVecWriter {
                self.0.extend_from_slice(buf);
                Ok(())
        }
-       fn size_hint(&mut self, size: usize) {
-               self.0.reserve_exact(size);
-       }
 }
 
 pub struct TestFeeEstimator {
@@ -349,7 +346,6 @@ impl msgs::RoutingMessageHandler for TestRoutingMessageHandler {
                self.chan_upds_recvd.fetch_add(1, Ordering::AcqRel);
                Err(msgs::LightningError { err: "".to_owned(), action: msgs::ErrorAction::IgnoreError })
        }
-       fn handle_htlc_fail_channel_update(&self, _update: &msgs::HTLCFailChannelUpdate) {}
        fn get_next_channel_announcements(&self, starting_point: u64, batch_amount: u8) -> Vec<(msgs::ChannelAnnouncement, Option<msgs::ChannelUpdate>, Option<msgs::ChannelUpdate>)> {
                let mut chan_anns = Vec::new();
                const TOTAL_UPDS: u64 = 100;