Remove the `ChannelManager::channel_state`
[rust-lightning] / lightning / src / ln / functional_test_utils.rs
index bc7f627758bf58a3591db58afd96640578de0015..bcfa571f9ac317f9eb4d47230fab4025fe3f4f45 100644 (file)
@@ -10,7 +10,7 @@
 //! A bunch of useful utilities for building networks of nodes and exchanging messages between
 //! nodes for functional tests.
 
-use crate::chain::{BestBlock, ChannelMonitorUpdateStatus, Confirm, Listen, Watch, keysinterface::KeysInterface};
+use crate::chain::{BestBlock, ChannelMonitorUpdateStatus, Confirm, Listen, Watch, keysinterface::EntropySource};
 use crate::chain::channelmonitor::ChannelMonitor;
 use crate::chain::transaction::OutPoint;
 use crate::ln::{PaymentPreimage, PaymentHash, PaymentSecret};
@@ -107,6 +107,14 @@ pub enum ConnectStyle {
        /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
        /// make a single `best_block_updated` call.
        TransactionsFirstSkippingBlocks,
+       /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
+       /// make a single `best_block_updated` call. Further, we call `transactions_confirmed` multiple
+       /// times to ensure it's idempotent.
+       TransactionsDuplicativelyFirstSkippingBlocks,
+       /// The same as `TransactionsFirst`, however when we have multiple blocks to connect, we only
+       /// make a single `best_block_updated` call. Further, we call `transactions_confirmed` multiple
+       /// times to ensure it's idempotent.
+       HighlyRedundantTransactionsFirstSkippingBlocks,
        /// The same as `TransactionsFirst` when connecting blocks. During disconnection only
        /// `transaction_unconfirmed` is called.
        TransactionsFirstReorgsOnlyTip,
@@ -121,14 +129,16 @@ impl ConnectStyle {
                        use core::hash::{BuildHasher, Hasher};
                        // Get a random value using the only std API to do so - the DefaultHasher
                        let rand_val = std::collections::hash_map::RandomState::new().build_hasher().finish();
-                       let res = match rand_val % 7 {
+                       let res = match rand_val % 9 {
                                0 => ConnectStyle::BestBlockFirst,
                                1 => ConnectStyle::BestBlockFirstSkippingBlocks,
                                2 => ConnectStyle::BestBlockFirstReorgsOnlyTip,
                                3 => ConnectStyle::TransactionsFirst,
                                4 => ConnectStyle::TransactionsFirstSkippingBlocks,
-                               5 => ConnectStyle::TransactionsFirstReorgsOnlyTip,
-                               6 => ConnectStyle::FullBlockViaListen,
+                               5 => ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks,
+                               6 => ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks,
+                               7 => ConnectStyle::TransactionsFirstReorgsOnlyTip,
+                               8 => ConnectStyle::FullBlockViaListen,
                                _ => unreachable!(),
                        };
                        eprintln!("Using Block Connection Style: {:?}", res);
@@ -143,6 +153,7 @@ impl ConnectStyle {
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
        let skip_intermediaries = match *node.connect_style.borrow() {
                ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
                        ConnectStyle::BestBlockFirstReorgsOnlyTip|ConnectStyle::TransactionsFirstReorgsOnlyTip => true,
                _ => false,
        };
@@ -193,8 +204,32 @@ fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: Block, sk
                                node.node.best_block_updated(&block.header, height);
                                node.node.transactions_confirmed(&block.header, &txdata, height);
                        },
-                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks|ConnectStyle::TransactionsFirstReorgsOnlyTip => {
+                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks|ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|
+                       ConnectStyle::TransactionsFirstReorgsOnlyTip => {
+                               if *node.connect_style.borrow() == ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks {
+                                       let mut connections = Vec::new();
+                                       for (block, height) in node.blocks.lock().unwrap().iter() {
+                                               if !block.txdata.is_empty() {
+                                                       // Reconnect all transactions we've ever seen to ensure transaction connection
+                                                       // is *really* idempotent. This is a somewhat likely deployment for some
+                                                       // esplora implementations of chain sync which try to reduce state and
+                                                       // complexity as much as possible.
+                                                       //
+                                                       // Sadly we have to clone the block here to maintain lockorder. In the
+                                                       // future we should consider Arc'ing the blocks to avoid this.
+                                                       connections.push((block.clone(), *height));
+                                               }
+                                       }
+                                       for (old_block, height) in connections {
+                                               node.chain_monitor.chain_monitor.transactions_confirmed(&old_block.header,
+                                                       &old_block.txdata.iter().enumerate().collect::<Vec<_>>(), height);
+                                       }
+                               }
                                node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               if *node.connect_style.borrow() == ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks {
+                                       node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               }
                                call_claimable_balances(node);
                                node.chain_monitor.chain_monitor.best_block_updated(&block.header, height);
                                node.node.transactions_confirmed(&block.header, &txdata, height);
@@ -226,7 +261,8 @@ pub fn disconnect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, count: u32)
                                node.chain_monitor.chain_monitor.block_disconnected(&orig.0.header, orig.1);
                                Listen::block_disconnected(node.node, &orig.0.header, orig.1);
                        },
-                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks|
+                       ConnectStyle::HighlyRedundantTransactionsFirstSkippingBlocks|ConnectStyle::TransactionsDuplicativelyFirstSkippingBlocks => {
                                if i == count - 1 {
                                        node.chain_monitor.chain_monitor.best_block_updated(&prev.0.header, prev.1);
                                        node.node.best_block_updated(&prev.0.header, prev.1);
@@ -265,10 +301,11 @@ pub struct NodeCfg<'a> {
        pub chain_source: &'a test_utils::TestChainSource,
        pub tx_broadcaster: &'a test_utils::TestBroadcaster,
        pub fee_estimator: &'a test_utils::TestFeeEstimator,
+       pub router: test_utils::TestRouter<'a>,
        pub chain_monitor: test_utils::TestChainMonitor<'a>,
        pub keys_manager: &'a test_utils::TestKeysInterface,
        pub logger: &'a test_utils::TestLogger,
-       pub network_graph: NetworkGraph<&'a test_utils::TestLogger>,
+       pub network_graph: Arc<NetworkGraph<&'a test_utils::TestLogger>>,
        pub node_seed: [u8; 32],
        pub features: InitFeatures,
 }
@@ -277,10 +314,11 @@ pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub chain_source: &'c test_utils::TestChainSource,
        pub tx_broadcaster: &'c test_utils::TestBroadcaster,
        pub fee_estimator: &'c test_utils::TestFeeEstimator,
+       pub router: &'b test_utils::TestRouter<'c>,
        pub chain_monitor: &'b test_utils::TestChainMonitor<'c>,
        pub keys_manager: &'b test_utils::TestKeysInterface,
-       pub node: &'a ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>,
-       pub network_graph: &'b NetworkGraph<&'c test_utils::TestLogger>,
+       pub node: &'a ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'b test_utils::TestRouter<'c>, &'c test_utils::TestLogger>,
+       pub network_graph: &'a NetworkGraph<&'c test_utils::TestLogger>,
        pub gossip_sync: P2PGossipSync<&'b NetworkGraph<&'c test_utils::TestLogger>, &'c test_utils::TestChainSource, &'c test_utils::TestLogger>,
        pub node_seed: [u8; 32],
        pub network_payment_count: Rc<RefCell<u8>>,
@@ -318,8 +356,8 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                                panic!("Had {} excess added monitors on node {}", added_monitors.len(), self.logger.id);
                        }
 
-                       // Check that if we serialize the Router, we can deserialize it again.
-                       {
+                       // Check that if we serialize the network graph, we can deserialize it again.
+                       let network_graph = {
                                let mut w = test_utils::TestVecWriter(Vec::new());
                                self.network_graph.write(&mut w).unwrap();
                                let network_graph_deser = <NetworkGraph<_>>::read(&mut io::Cursor::new(&w.0), self.logger).unwrap();
@@ -347,7 +385,8 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                                                None => break,
                                        };
                                }
-                       }
+                               network_graph_deser
+                       };
 
                        // Check that if we serialize and then deserialize all our channel monitors we get the
                        // same set of outputs to watch for on chain as we have now. Note that if we write
@@ -379,10 +418,11 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
 
                                let mut w = test_utils::TestVecWriter(Vec::new());
                                self.node.write(&mut w).unwrap();
-                               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(w.0), ChannelManagerReadArgs {
+                               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(w.0), ChannelManagerReadArgs {
                                        default_config: *self.node.get_current_default_configuration(),
                                        keys_manager: self.keys_manager,
                                        fee_estimator: &test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) },
+                                       router: &test_utils::TestRouter::new(Arc::new(network_graph)),
                                        chain_monitor: self.chain_monitor,
                                        tx_broadcaster: &broadcaster,
                                        logger: &self.logger,
@@ -518,22 +558,102 @@ macro_rules! get_htlc_update_msgs {
        }
 }
 
+/// Fetches the first `msg_event` to the passed `node_id` in the passed `msg_events` vec.
+/// Returns the `msg_event`, along with an updated `msg_events` vec with the message removed.
+///
+/// Note that even though `BroadcastChannelAnnouncement` and `BroadcastChannelUpdate`
+/// `msg_events` are stored under specific peers, this function does not fetch such `msg_events` as
+/// such messages are intended to all peers.
+pub fn remove_first_msg_event_to_node(msg_node_id: &PublicKey, msg_events: &Vec<MessageSendEvent>) -> (MessageSendEvent, Vec<MessageSendEvent>) {
+       let ev_index = msg_events.iter().position(|e| { match e {
+               MessageSendEvent::SendAcceptChannel { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendOpenChannel { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendFundingCreated { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendFundingSigned { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendChannelReady { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendAnnouncementSignatures { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::UpdateHTLCs { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendRevokeAndACK { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendClosingSigned { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendShutdown { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendChannelReestablish { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendChannelAnnouncement { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::BroadcastChannelAnnouncement { .. } => {
+                       false
+               },
+               MessageSendEvent::BroadcastChannelUpdate { .. } => {
+                       false
+               },
+               MessageSendEvent::SendChannelUpdate { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::HandleError { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendChannelRangeQuery { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendShortIdsQuery { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendReplyChannelRange { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+               MessageSendEvent::SendGossipTimestampFilter { node_id, .. } => {
+                       node_id == msg_node_id
+               },
+       }});
+       if ev_index.is_some() {
+               let mut updated_msg_events = msg_events.to_vec();
+               let ev = updated_msg_events.remove(ev_index.unwrap());
+               (ev, updated_msg_events)
+       } else {
+               panic!("Couldn't find any MessageSendEvent to the node!")
+       }
+}
+
 #[cfg(test)]
 macro_rules! get_channel_ref {
-       ($node: expr, $lock: ident, $channel_id: expr) => {
+       ($node: expr, $counterparty_node: expr, $per_peer_state_lock: ident, $peer_state_lock: ident, $channel_id: expr) => {
                {
-                       $lock = $node.node.channel_state.lock().unwrap();
-                       $lock.by_id.get_mut(&$channel_id).unwrap()
+                       $per_peer_state_lock = $node.node.per_peer_state.read().unwrap();
+                       $peer_state_lock = $per_peer_state_lock.get(&$counterparty_node.node.get_our_node_id()).unwrap().lock().unwrap();
+                       $peer_state_lock.channel_by_id.get_mut(&$channel_id).unwrap()
                }
        }
 }
 
 #[cfg(test)]
 macro_rules! get_feerate {
-       ($node: expr, $channel_id: expr) => {
+       ($node: expr, $counterparty_node: expr, $channel_id: expr) => {
                {
-                       let mut lock;
-                       let chan = get_channel_ref!($node, lock, $channel_id);
+                       let mut per_peer_state_lock;
+                       let mut peer_state_lock;
+                       let chan = get_channel_ref!($node, $counterparty_node, per_peer_state_lock, peer_state_lock, $channel_id);
                        chan.get_feerate()
                }
        }
@@ -541,10 +661,11 @@ macro_rules! get_feerate {
 
 #[cfg(test)]
 macro_rules! get_opt_anchors {
-       ($node: expr, $channel_id: expr) => {
+       ($node: expr, $counterparty_node: expr, $channel_id: expr) => {
                {
-                       let mut lock;
-                       let chan = get_channel_ref!($node, lock, $channel_id);
+                       let mut per_peer_state_lock;
+                       let mut peer_state_lock;
+                       let chan = get_channel_ref!($node, $counterparty_node, per_peer_state_lock, peer_state_lock, $channel_id);
                        chan.opt_anchors()
                }
        }
@@ -619,7 +740,7 @@ macro_rules! check_added_monitors {
        }
 }
 
-pub fn _reload_node<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, default_config: UserConfig, chanman_encoded: &[u8], monitors_encoded: &[&[u8]]) -> ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger> {
+pub fn _reload_node<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, default_config: UserConfig, chanman_encoded: &[u8], monitors_encoded: &[&[u8]]) -> ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'b test_utils::TestRouter<'c>, &'c test_utils::TestLogger> {
        let mut monitors_read = Vec::with_capacity(monitors_encoded.len());
        for encoded in monitors_encoded {
                let mut monitor_read = &encoded[..];
@@ -635,10 +756,11 @@ pub fn _reload_node<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, default_config:
                for monitor in monitors_read.iter_mut() {
                        assert!(channel_monitors.insert(monitor.get_funding_txo().0, monitor).is_none());
                }
-               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut node_read, ChannelManagerReadArgs {
+               <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut node_read, ChannelManagerReadArgs {
                        default_config,
                        keys_manager: node.keys_manager,
                        fee_estimator: node.fee_estimator,
+                       router: node.router,
                        chain_monitor: node.chain_monitor,
                        tx_broadcaster: node.tx_broadcaster,
                        logger: node.logger,
@@ -1232,13 +1354,14 @@ macro_rules! commitment_signed_dance {
                        let (bs_revoke_and_ack, extra_msg_option) = {
                                let events = $node_b.node.get_and_clear_pending_msg_events();
                                assert!(events.len() <= 2);
-                               (match events[0] {
+                               let (node_a_event, events) = remove_first_msg_event_to_node(&$node_a.node.get_our_node_id(), &events);
+                               (match node_a_event {
                                        MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
                                                assert_eq!(*node_id, $node_a.node.get_our_node_id());
                                                (*msg).clone()
                                        },
                                        _ => panic!("Unexpected event"),
-                               }, events.get(1).map(|e| e.clone()))
+                               }, events.get(0).map(|e| e.clone()))
                        };
                        check_added_monitors!($node_b, 1);
                        if $fail_backwards {
@@ -1279,11 +1402,20 @@ macro_rules! commitment_signed_dance {
                                expect_pending_htlcs_forwardable_and_htlc_handling_failed!($node_a, vec![$crate::util::events::HTLCDestination::NextHopChannel{ node_id: Some($node_b.node.get_our_node_id()), channel_id: $commitment_signed.channel_id }]);
                                check_added_monitors!($node_a, 1);
 
-                               let channel_state = $node_a.node.channel_state.lock().unwrap();
-                               assert_eq!(channel_state.pending_msg_events.len(), 1);
-                               if let MessageSendEvent::UpdateHTLCs { ref node_id, .. } = channel_state.pending_msg_events[0] {
-                                       assert_ne!(*node_id, $node_b.node.get_our_node_id());
-                               } else { panic!("Unexpected event"); }
+                               let node_a_per_peer_state = $node_a.node.per_peer_state.read().unwrap();
+                               let mut number_of_msg_events = 0;
+                               for (cp_id, peer_state_mutex) in node_a_per_peer_state.iter() {
+                                       let peer_state = peer_state_mutex.lock().unwrap();
+                                       let cp_pending_msg_events = &peer_state.pending_msg_events;
+                                       number_of_msg_events += cp_pending_msg_events.len();
+                                       if cp_pending_msg_events.len() == 1 {
+                                               if let MessageSendEvent::UpdateHTLCs { .. } = cp_pending_msg_events[0] {
+                                                       assert_ne!(*cp_id, $node_b.node.get_our_node_id());
+                                               } else { panic!("Unexpected event"); }
+                                       }
+                               }
+                               // Expecting the failure backwards event to the previous hop (not `node_b`)
+                               assert_eq!(number_of_msg_events, 1);
                        } else {
                                assert!($node_a.node.get_and_clear_pending_msg_events().is_empty());
                        }
@@ -1316,7 +1448,7 @@ macro_rules! get_payment_preimage_hash {
 #[macro_export]
 macro_rules! get_route {
        ($send_node: expr, $payment_params: expr, $recv_value: expr, $cltv: expr) => {{
-               use $crate::chain::keysinterface::KeysInterface;
+               use $crate::chain::keysinterface::EntropySource;
                let scorer = $crate::util::test_utils::TestScorer::with_penalty(0);
                let keys_manager = $crate::util::test_utils::TestKeysInterface::new(&[0u8; 32], bitcoin::network::constants::Network::Testnet);
                let random_seed_bytes = keys_manager.get_secure_random_bytes();
@@ -1433,20 +1565,20 @@ macro_rules! expect_pending_htlcs_forwardable_from_events {
                }
        }}
 }
-
 #[macro_export]
 #[cfg(any(test, feature = "_bench_unstable", feature = "_test_utils"))]
-macro_rules! expect_payment_received {
+macro_rules! expect_payment_claimable {
        ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
-               expect_payment_received!($node, $expected_payment_hash, $expected_payment_secret, $expected_recv_value, None)
+               expect_payment_claimable!($node, $expected_payment_hash, $expected_payment_secret, $expected_recv_value, None, $node.node.get_our_node_id())
        };
-       ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr, $expected_payment_preimage: expr) => {
+       ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr, $expected_payment_preimage: expr, $expected_receiver_node_id: expr) => {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       $crate::util::events::Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+                       $crate::util::events::Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, via_channel_id: _, via_user_channel_id: _ } => {
                                assert_eq!($expected_payment_hash, *payment_hash);
                                assert_eq!($expected_recv_value, amount_msat);
+                               assert_eq!($expected_receiver_node_id, receiver_node_id.unwrap());
                                match purpose {
                                        $crate::util::events::PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                assert_eq!(&$expected_payment_preimage, payment_preimage);
@@ -1720,7 +1852,7 @@ pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>,
        payment_id
 }
 
-pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) {
+pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, clear_recipient_events: bool, expected_preimage: Option<PaymentPreimage>) {
        let mut payment_event = SendEvent::from_event(ev);
        let mut prev_node = origin_node;
 
@@ -1735,11 +1867,12 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
 
                if idx == expected_path.len() - 1 && clear_recipient_events {
                        let events_2 = node.node.get_and_clear_pending_events();
-                       if payment_received_expected {
+                       if payment_claimable_expected {
                                assert_eq!(events_2.len(), 1);
                                match events_2[0] {
-                                       Event::PaymentReceived { ref payment_hash, ref purpose, amount_msat } => {
+                                       Event::PaymentClaimable { ref payment_hash, ref purpose, amount_msat, receiver_node_id, ref via_channel_id, ref via_user_channel_id } => {
                                                assert_eq!(our_payment_hash, *payment_hash);
+                                               assert_eq!(node.node.get_our_node_id(), receiver_node_id.unwrap());
                                                match &purpose {
                                                        PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
                                                                assert_eq!(expected_preimage, *payment_preimage);
@@ -1751,6 +1884,8 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
                                                        },
                                                }
                                                assert_eq!(amount_msat, recv_value);
+                                               assert!(node.node.list_channels().iter().any(|details| details.channel_id == via_channel_id.unwrap()));
+                                               assert!(node.node.list_channels().iter().any(|details| details.user_channel_id == via_user_channel_id.unwrap()));
                                        },
                                        _ => panic!("Unexpected event"),
                                }
@@ -1769,16 +1904,18 @@ pub fn do_pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_p
        }
 }
 
-pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool, expected_preimage: Option<PaymentPreimage>) {
-       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_received_expected, true, expected_preimage);
+pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_claimable_expected: bool, expected_preimage: Option<PaymentPreimage>) {
+       do_pass_along_path(origin_node, expected_path, recv_value, our_payment_hash, our_payment_secret, ev, payment_claimable_expected, true, expected_preimage);
 }
 
 pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
        let mut events = origin_node.node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), expected_route.len());
-       for (path_idx, (ev, expected_path)) in events.drain(..).zip(expected_route.iter()).enumerate() {
+       for (path_idx, expected_path) in expected_route.iter().enumerate() {
+               let (ev, updated_events) = remove_first_msg_event_to_node(&expected_path[0].node.get_our_node_id(), &events);
+               events = updated_events;
                // Once we've gotten through all the HTLCs, the last one should result in a
-               // PaymentReceived (but each previous one should not!), .
+               // PaymentClaimable (but each previous one should not!), .
                let expect_payment = path_idx == expected_route.len() - 1;
                pass_along_path(origin_node, expected_path, recv_value, our_payment_hash.clone(), Some(our_payment_secret), ev, expect_payment, None);
        }
@@ -1827,10 +1964,18 @@ pub fn do_claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>,
                }
        }
        let mut per_path_msgs: Vec<((msgs::UpdateFulfillHTLC, msgs::CommitmentSigned), PublicKey)> = Vec::with_capacity(expected_paths.len());
-       let events = expected_paths[0].last().unwrap().node.get_and_clear_pending_msg_events();
+       let mut events = expected_paths[0].last().unwrap().node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), expected_paths.len());
-       for ev in events.iter() {
-               per_path_msgs.push(msgs_from_ev!(ev));
+
+       if events.len() == 1 {
+               per_path_msgs.push(msgs_from_ev!(&events[0]));
+       } else {
+               for expected_path in expected_paths.iter() {
+                       // For MPP payments, we always want the message to the first node in the path.
+                       let (ev, updated_events) = remove_first_msg_event_to_node(&expected_path[0].node.get_our_node_id(), &events);
+                       per_path_msgs.push(msgs_from_ev!(&ev));
+                       events = updated_events;
+               }
        }
 
        for (expected_route, (path_msgs, next_hop)) in expected_paths.iter().zip(per_path_msgs.drain(..)) {
@@ -1852,9 +1997,10 @@ pub fn do_claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>,
                                {
                                        $node.node.handle_update_fulfill_htlc(&$prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
                                        let fee = {
-                                               let channel_state = $node.node.channel_state.lock().unwrap();
-                                               let channel = channel_state
-                                                       .by_id.get(&next_msgs.as_ref().unwrap().0.channel_id).unwrap();
+                                               let per_peer_state = $node.node.per_peer_state.read().unwrap();
+                                               let peer_state = per_peer_state.get(&$prev_node.node.get_our_node_id())
+                                                       .unwrap().lock().unwrap();
+                                               let channel = peer_state.channel_by_id.get(&next_msgs.as_ref().unwrap().0.channel_id).unwrap();
                                                if let Some(prev_config) = channel.prev_config() {
                                                        prev_config.forwarding_fee_base_msat
                                                } else {
@@ -2113,7 +2259,7 @@ pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
                let seed = [i as u8; 32];
                let keys_manager = test_utils::TestKeysInterface::new(&seed, Network::Testnet);
 
-               chan_mon_cfgs.push(TestChanMonCfg{ tx_broadcaster, fee_estimator, chain_source, logger, persister, keys_manager });
+               chan_mon_cfgs.push(TestChanMonCfg { tx_broadcaster, fee_estimator, chain_source, logger, persister, keys_manager });
        }
 
        chan_mon_cfgs
@@ -2124,17 +2270,19 @@ pub fn create_node_cfgs<'a>(node_count: usize, chanmon_cfgs: &'a Vec<TestChanMon
 
        for i in 0..node_count {
                let chain_monitor = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[i].chain_source), &chanmon_cfgs[i].tx_broadcaster, &chanmon_cfgs[i].logger, &chanmon_cfgs[i].fee_estimator, &chanmon_cfgs[i].persister, &chanmon_cfgs[i].keys_manager);
+               let network_graph = Arc::new(NetworkGraph::new(chanmon_cfgs[i].chain_source.genesis_hash, &chanmon_cfgs[i].logger));
                let seed = [i as u8; 32];
                nodes.push(NodeCfg {
                        chain_source: &chanmon_cfgs[i].chain_source,
                        logger: &chanmon_cfgs[i].logger,
                        tx_broadcaster: &chanmon_cfgs[i].tx_broadcaster,
                        fee_estimator: &chanmon_cfgs[i].fee_estimator,
+                       router: test_utils::TestRouter::new(network_graph.clone()),
                        chain_monitor,
                        keys_manager: &chanmon_cfgs[i].keys_manager,
                        node_seed: seed,
                        features: channelmanager::provided_init_features(),
-                       network_graph: NetworkGraph::new(chanmon_cfgs[i].chain_source.genesis_hash, &chanmon_cfgs[i].logger),
+                       network_graph,
                });
        }
 
@@ -2157,7 +2305,7 @@ pub fn test_default_channel_config() -> UserConfig {
        default_config
 }
 
-pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>>, node_config: &[Option<UserConfig>]) -> Vec<ChannelManager<&'a TestChainMonitor<'b>, &'b test_utils::TestBroadcaster, &'a test_utils::TestKeysInterface, &'b test_utils::TestFeeEstimator, &'b test_utils::TestLogger>> {
+pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>>, node_config: &[Option<UserConfig>]) -> Vec<ChannelManager<&'a TestChainMonitor<'b>, &'b test_utils::TestBroadcaster, &'a test_utils::TestKeysInterface, &'b test_utils::TestFeeEstimator, &'a test_utils::TestRouter<'b>, &'b test_utils::TestLogger>> {
        let mut chanmgrs = Vec::new();
        for i in 0..node_count {
                let network = Network::Testnet;
@@ -2165,7 +2313,7 @@ pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>
                        network,
                        best_block: BestBlock::from_genesis(network),
                };
-               let node = ChannelManager::new(cfgs[i].fee_estimator, &cfgs[i].chain_monitor, cfgs[i].tx_broadcaster, cfgs[i].logger, cfgs[i].keys_manager,
+               let node = ChannelManager::new(cfgs[i].fee_estimator, &cfgs[i].chain_monitor, cfgs[i].tx_broadcaster, &cfgs[i].router, cfgs[i].logger, cfgs[i].keys_manager,
                        if node_config[i].is_some() { node_config[i].clone().unwrap() } else { test_default_channel_config() }, params);
                chanmgrs.push(node);
        }
@@ -2173,19 +2321,19 @@ pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>
        chanmgrs
 }
 
-pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeCfg<'c>>, chan_mgrs: &'a Vec<ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestLogger>>) -> Vec<Node<'a, 'b, 'c>> {
+pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeCfg<'c>>, chan_mgrs: &'a Vec<ChannelManager<&'b TestChainMonitor<'c>, &'c test_utils::TestBroadcaster, &'b test_utils::TestKeysInterface, &'c test_utils::TestFeeEstimator, &'c test_utils::TestRouter, &'c test_utils::TestLogger>>) -> Vec<Node<'a, 'b, 'c>> {
        let mut nodes = Vec::new();
        let chan_count = Rc::new(RefCell::new(0));
        let payment_count = Rc::new(RefCell::new(0));
        let connect_style = Rc::new(RefCell::new(ConnectStyle::random_style()));
 
        for i in 0..node_count {
-               let gossip_sync = P2PGossipSync::new(&cfgs[i].network_graph, None, cfgs[i].logger);
+               let gossip_sync = P2PGossipSync::new(cfgs[i].network_graph.as_ref(), None, cfgs[i].logger);
                nodes.push(Node{
                        chain_source: cfgs[i].chain_source, tx_broadcaster: cfgs[i].tx_broadcaster,
-                       fee_estimator: cfgs[i].fee_estimator,
+                       fee_estimator: cfgs[i].fee_estimator, router: &cfgs[i].router,
                        chain_monitor: &cfgs[i].chain_monitor, keys_manager: &cfgs[i].keys_manager,
-                       node: &chan_mgrs[i], network_graph: &cfgs[i].network_graph, gossip_sync,
+                       node: &chan_mgrs[i], network_graph: cfgs[i].network_graph.as_ref(), gossip_sync,
                        node_seed: cfgs[i].node_seed, network_chan_count: chan_count.clone(),
                        network_payment_count: payment_count.clone(), logger: cfgs[i].logger,
                        blocks: Arc::clone(&cfgs[i].tx_broadcaster.blocks),
@@ -2359,9 +2507,10 @@ pub fn get_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b,
 
 #[cfg(test)]
 macro_rules! get_channel_value_stat {
-       ($node: expr, $channel_id: expr) => {{
-               let chan_lock = $node.node.channel_state.lock().unwrap();
-               let chan = chan_lock.by_id.get(&$channel_id).unwrap();
+       ($node: expr, $counterparty_node: expr, $channel_id: expr) => {{
+               let peer_state_lock = $node.node.per_peer_state.read().unwrap();
+               let chan_lock = peer_state_lock.get(&$counterparty_node.node.get_our_node_id()).unwrap().lock().unwrap();
+               let chan = chan_lock.channel_by_id.get(&$channel_id).unwrap();
                chan.get_value_stat()
        }}
 }
@@ -2413,6 +2562,14 @@ macro_rules! handle_chan_reestablish_msgs {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                        }
 
+                       let mut had_channel_update = false; // ChannelUpdate may be now or later, but not both
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                               assert_eq!(*node_id, $dst_node.node.get_our_node_id());
+                               idx += 1;
+                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
+                               had_channel_update = true;
+                       }
+
                        let mut revoke_and_ack = None;
                        let mut commitment_update = None;
                        let order = if let Some(ev) = msg_events.get(idx) {
@@ -2457,6 +2614,7 @@ macro_rules! handle_chan_reestablish_msgs {
                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                idx += 1;
                                assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
+                               assert!(!had_channel_update);
                        }
 
                        assert_eq!(msg_events.len(), idx);