Merge pull request #949 from TheBlueMatt/2021-06-send-priv-update
[rust-lightning] / lightning / src / ln / functional_test_utils.rs
index e7f61c6a3dfeefb23f845b7a47d6ab0efa5abf60..2e9439916df7b8c64760813d94685621183dba01 100644 (file)
 //! A bunch of useful utilities for building networks of nodes and exchanging messages between
 //! nodes for functional tests.
 
-use chain::Watch;
+use chain::{BestBlock, Confirm, Listen, Watch};
 use chain::channelmonitor::ChannelMonitor;
 use chain::transaction::OutPoint;
-use ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure};
+use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
+use ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure};
 use routing::router::{Route, get_route};
 use routing::network_graph::{NetGraphMsgHandler, NetworkGraph};
-use ln::features::InitFeatures;
+use ln::features::{InitFeatures, InvoiceFeatures};
 use ln::msgs;
 use ln::msgs::{ChannelMessageHandler,RoutingMessageHandler};
 use util::enforcing_trait_impls::EnforcingSigner;
 use util::test_utils;
 use util::test_utils::TestChainMonitor;
-use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
 use util::errors::APIError;
 use util::config::UserConfig;
 use util::ser::{ReadableArgs, Writeable, Readable};
@@ -38,11 +39,11 @@ use bitcoin::hash_types::BlockHash;
 
 use bitcoin::secp256k1::key::PublicKey;
 
-use std::cell::RefCell;
+use prelude::*;
+use core::cell::RefCell;
 use std::rc::Rc;
-use std::sync::Mutex;
-use std::mem;
-use std::collections::HashMap;
+use std::sync::{Arc, Mutex};
+use core::mem;
 
 pub const CHAN_CONFIRM_DEPTH: u32 = 10;
 
@@ -60,21 +61,15 @@ pub fn mine_transaction<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &Transac
 /// Mine the given transaction at the given height, mining blocks as required to build to that
 /// height
 pub fn confirm_transaction_at<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &Transaction, conf_height: u32) {
-       let starting_block = node.best_block_info();
+       let first_connect_height = node.best_block_info().1 + 1;
+       assert!(first_connect_height <= conf_height);
+       if conf_height > first_connect_height {
+               connect_blocks(node, conf_height - first_connect_height);
+       }
        let mut block = Block {
-               header: BlockHeader { version: 0x20000000, prev_blockhash: starting_block.0, merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
+               header: BlockHeader { version: 0x20000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: conf_height, bits: 42, nonce: 42 },
                txdata: Vec::new(),
        };
-       let height = starting_block.1 + 1;
-       assert!(height <= conf_height);
-       for _ in height..conf_height {
-               connect_block(node, &block);
-               block = Block {
-                       header: BlockHeader { version: 0x20000000, prev_blockhash: block.header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
-                       txdata: vec![],
-               };
-       }
-
        for _ in 0..*node.network_chan_count.borrow() { // Make sure we don't end up with channels at the same short id by offsetting by chan_count
                block.txdata.push(Transaction { version: 0, lock_time: 0, input: Vec::new(), output: Vec::new() });
        }
@@ -82,42 +77,107 @@ pub fn confirm_transaction_at<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, tx: &T
        connect_block(node, &block);
 }
 
+/// The possible ways we may notify a ChannelManager of a new block
+#[derive(Clone, Copy, PartialEq)]
+pub enum ConnectStyle {
+       /// Calls best_block_updated first, detecting transactions in the block only after receiving the
+       /// header and height information.
+       BestBlockFirst,
+       /// The same as BestBlockFirst, however when we have multiple blocks to connect, we only
+       /// make a single best_block_updated call.
+       BestBlockFirstSkippingBlocks,
+       /// Calls transactions_confirmed first, detecting transactions in the block before updating the
+       /// header and height information.
+       TransactionsFirst,
+       /// The same as TransactionsFirst, however when we have multiple blocks to connect, we only
+       /// make a single best_block_updated call.
+       TransactionsFirstSkippingBlocks,
+       /// Provides the full block via the chain::Listen interface. In the current code this is
+       /// equivalent to TransactionsFirst with some additional assertions.
+       FullBlockViaListen,
+}
+
 pub fn connect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, depth: u32) -> BlockHash {
+       let skip_intermediaries = match *node.connect_style.borrow() {
+               ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => true,
+               _ => false,
+       };
+
+       let height = node.best_block_info().1 + 1;
        let mut block = Block {
-               header: BlockHeader { version: 0x2000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
+               header: BlockHeader { version: 0x2000000, prev_blockhash: node.best_block_hash(), merkle_root: Default::default(), time: height, bits: 42, nonce: 42 },
                txdata: vec![],
        };
-       connect_block(node, &block);
-       for _ in 2..depth + 1 {
+       assert!(depth >= 1);
+       for i in 1..depth {
+               do_connect_block(node, &block, skip_intermediaries);
                block = Block {
-                       header: BlockHeader { version: 0x20000000, prev_blockhash: block.header.block_hash(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 },
+                       header: BlockHeader { version: 0x20000000, prev_blockhash: block.header.block_hash(), merkle_root: Default::default(), time: height + i, bits: 42, nonce: 42 },
                        txdata: vec![],
                };
-               connect_block(node, &block);
        }
+       connect_block(node, &block);
        block.header.block_hash()
 }
 
 pub fn connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: &Block) {
-       let txdata: Vec<_> = block.txdata.iter().enumerate().collect();
+       do_connect_block(node, block, false);
+}
+
+fn do_connect_block<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, block: &Block, skip_intermediaries: bool) {
        let height = node.best_block_info().1 + 1;
-       node.chain_monitor.chain_monitor.block_connected(&block.header, &txdata, height);
-       node.node.block_connected(&block.header, &txdata, height);
+       if !skip_intermediaries {
+               let txdata: Vec<_> = block.txdata.iter().enumerate().collect();
+               match *node.connect_style.borrow() {
+                       ConnectStyle::BestBlockFirst|ConnectStyle::BestBlockFirstSkippingBlocks => {
+                               node.chain_monitor.chain_monitor.best_block_updated(&block.header, height);
+                               node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               node.node.best_block_updated(&block.header, height);
+                               node.node.transactions_confirmed(&block.header, &txdata, height);
+                       },
+                       ConnectStyle::TransactionsFirst|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                               node.chain_monitor.chain_monitor.transactions_confirmed(&block.header, &txdata, height);
+                               node.chain_monitor.chain_monitor.best_block_updated(&block.header, height);
+                               node.node.transactions_confirmed(&block.header, &txdata, height);
+                               node.node.best_block_updated(&block.header, height);
+                       },
+                       ConnectStyle::FullBlockViaListen => {
+                               node.chain_monitor.chain_monitor.block_connected(&block, height);
+                               node.node.block_connected(&block, height);
+                       }
+               }
+       }
        node.node.test_process_background_events();
-       node.blocks.borrow_mut().push((block.header, height));
+       node.blocks.lock().unwrap().push((block.header, height));
 }
 
 pub fn disconnect_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>, count: u32) {
-       for _ in 0..count {
-               let orig_header = node.blocks.borrow_mut().pop().unwrap();
+       for i in 0..count {
+               let orig_header = node.blocks.lock().unwrap().pop().unwrap();
                assert!(orig_header.1 > 0); // Cannot disconnect genesis
-               node.chain_monitor.chain_monitor.block_disconnected(&orig_header.0, orig_header.1);
-               node.node.block_disconnected(&orig_header.0);
+               let prev_header = node.blocks.lock().unwrap().last().unwrap().clone();
+
+               match *node.connect_style.borrow() {
+                       ConnectStyle::FullBlockViaListen => {
+                               node.chain_monitor.chain_monitor.block_disconnected(&orig_header.0, orig_header.1);
+                               Listen::block_disconnected(node.node, &orig_header.0, orig_header.1);
+                       },
+                       ConnectStyle::BestBlockFirstSkippingBlocks|ConnectStyle::TransactionsFirstSkippingBlocks => {
+                               if i == count - 1 {
+                                       node.chain_monitor.chain_monitor.best_block_updated(&prev_header.0, prev_header.1);
+                                       node.node.best_block_updated(&prev_header.0, prev_header.1);
+                               }
+                       },
+                       _ => {
+                               node.chain_monitor.chain_monitor.best_block_updated(&prev_header.0, prev_header.1);
+                               node.node.best_block_updated(&prev_header.0, prev_header.1);
+                       },
+               }
        }
 }
 
 pub fn disconnect_all_blocks<'a, 'b, 'c, 'd>(node: &'a Node<'b, 'c, 'd>) {
-       let count = node.blocks.borrow_mut().len() as u32 - 1;
+       let count = node.blocks.lock().unwrap().len() as u32 - 1;
        disconnect_blocks(node, count);
 }
 
@@ -151,14 +211,18 @@ pub struct Node<'a, 'b: 'a, 'c: 'b> {
        pub network_payment_count: Rc<RefCell<u8>>,
        pub network_chan_count: Rc<RefCell<u32>>,
        pub logger: &'c test_utils::TestLogger,
-       pub blocks: RefCell<Vec<(BlockHeader, u32)>>,
+       pub blocks: Arc<Mutex<Vec<(BlockHeader, u32)>>>,
+       pub connect_style: Rc<RefCell<ConnectStyle>>,
 }
 impl<'a, 'b, 'c> Node<'a, 'b, 'c> {
        pub fn best_block_hash(&self) -> BlockHash {
-               self.blocks.borrow_mut().last().unwrap().0.block_hash()
+               self.blocks.lock().unwrap().last().unwrap().0.block_hash()
        }
        pub fn best_block_info(&self) -> (BlockHash, u32) {
-               self.blocks.borrow_mut().last().map(|(a, b)| (a.block_hash(), *b)).unwrap()
+               self.blocks.lock().unwrap().last().map(|(a, b)| (a.block_hash(), *b)).unwrap()
+       }
+       pub fn get_block_header(&self, height: u32) -> BlockHeader {
+               self.blocks.lock().unwrap()[height as usize].0
        }
 }
 
@@ -229,12 +293,13 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                                let mut w = test_utils::TestVecWriter(Vec::new());
                                self.node.write(&mut w).unwrap();
                                <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut ::std::io::Cursor::new(w.0), ChannelManagerReadArgs {
-                                       default_config: UserConfig::default(),
+                                       default_config: *self.node.get_current_default_configuration(),
                                        keys_manager: self.keys_manager,
                                        fee_estimator: &test_utils::TestFeeEstimator { sat_per_kw: 253 },
                                        chain_monitor: self.chain_monitor,
                                        tx_broadcaster: &test_utils::TestBroadcaster {
-                                               txn_broadcasted: Mutex::new(self.tx_broadcaster.txn_broadcasted.lock().unwrap().clone())
+                                               txn_broadcasted: Mutex::new(self.tx_broadcaster.txn_broadcasted.lock().unwrap().clone()),
+                                               blocks: Arc::new(Mutex::new(self.tx_broadcaster.blocks.lock().unwrap().clone())),
                                        },
                                        logger: &test_utils::TestLogger::new(),
                                        channel_monitors,
@@ -243,7 +308,8 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
 
                        let persister = test_utils::TestPersister::new();
                        let broadcaster = test_utils::TestBroadcaster {
-                               txn_broadcasted: Mutex::new(self.tx_broadcaster.txn_broadcasted.lock().unwrap().clone())
+                               txn_broadcasted: Mutex::new(self.tx_broadcaster.txn_broadcasted.lock().unwrap().clone()),
+                               blocks: Arc::new(Mutex::new(self.tx_broadcaster.blocks.lock().unwrap().clone())),
                        };
                        let chain_source = test_utils::TestChainSource::new(Network::Testnet);
                        let chain_monitor = test_utils::TestChainMonitor::new(Some(&chain_source), &broadcaster, &self.logger, &feeest, &persister, &self.keys_manager);
@@ -313,6 +379,24 @@ macro_rules! get_event_msg {
        }
 }
 
+/// Get a specific event from the pending events queue.
+#[macro_export]
+macro_rules! get_event {
+       ($node: expr, $event_type: path) => {
+               {
+                       let mut events = $node.node.get_and_clear_pending_events();
+                       assert_eq!(events.len(), 1);
+                       let ev = events.pop().unwrap();
+                       match ev {
+                               $event_type { .. } => {
+                                       ev
+                               },
+                               _ => panic!("Unexpected event"),
+                       }
+               }
+       }
+}
+
 #[cfg(test)]
 macro_rules! get_htlc_update_msgs {
        ($node: expr, $node_id: expr) => {
@@ -341,7 +425,8 @@ macro_rules! get_feerate {
        }
 }
 
-#[cfg(test)]
+/// Returns any local commitment transactions for the channel.
+#[macro_export]
 macro_rules! get_local_commitment_txn {
        ($node: expr, $channel_id: expr) => {
                {
@@ -421,7 +506,7 @@ pub fn create_chan_between_nodes_with_value_init<'a, 'b, 'c>(node_a: &Node<'a, '
 
        let (temporary_channel_id, tx, funding_output) = create_funding_transaction(node_a, channel_value, 42);
 
-       node_a.node.funding_transaction_generated(&temporary_channel_id, funding_output);
+       node_a.node.funding_transaction_generated(&temporary_channel_id, tx.clone()).unwrap();
        check_added_monitors!(node_a, 0);
 
        node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
@@ -441,14 +526,11 @@ pub fn create_chan_between_nodes_with_value_init<'a, 'b, 'c>(node_a: &Node<'a, '
        }
 
        let events_4 = node_a.node.get_and_clear_pending_events();
-       assert_eq!(events_4.len(), 1);
-       match events_4[0] {
-               Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
-                       assert_eq!(user_channel_id, 42);
-                       assert_eq!(*funding_txo, funding_output);
-               },
-               _ => panic!("Unexpected event"),
-       };
+       assert_eq!(events_4.len(), 0);
+
+       assert_eq!(node_a.tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
+       assert_eq!(node_a.tx_broadcaster.txn_broadcasted.lock().unwrap()[0], tx);
+       node_a.tx_broadcaster.txn_broadcasted.lock().unwrap().clear();
 
        tx
 }
@@ -480,7 +562,7 @@ pub fn create_chan_between_nodes_with_value_confirm_second<'a, 'b, 'c>(node_recv
 }
 
 pub fn create_chan_between_nodes_with_value_confirm<'a, 'b, 'c, 'd>(node_a: &'a Node<'b, 'c, 'd>, node_b: &'a Node<'b, 'c, 'd>, tx: &Transaction) -> ((msgs::FundingLocked, msgs::AnnouncementSignatures), [u8; 32]) {
-       let conf_height = std::cmp::max(node_a.best_block_info().1 + 1, node_b.best_block_info().1 + 1);
+       let conf_height = core::cmp::max(node_a.best_block_info().1 + 1, node_b.best_block_info().1 + 1);
        create_chan_between_nodes_with_value_confirm_first(node_a, node_b, tx, conf_height);
        confirm_transaction_at(node_a, tx, conf_height);
        connect_blocks(node_a, CHAN_CONFIRM_DEPTH - 1);
@@ -538,8 +620,22 @@ pub fn create_announced_chan_between_nodes_with_value<'a, 'b, 'c, 'd>(nodes: &'a
 pub fn update_nodes_with_chan_announce<'a, 'b, 'c, 'd>(nodes: &'a Vec<Node<'b, 'c, 'd>>, a: usize, b: usize, ann: &msgs::ChannelAnnouncement, upd_1: &msgs::ChannelUpdate, upd_2: &msgs::ChannelUpdate) {
        nodes[a].node.broadcast_node_announcement([0, 0, 0], [0; 32], Vec::new());
        let a_events = nodes[a].node.get_and_clear_pending_msg_events();
-       assert_eq!(a_events.len(), 1);
-       let a_node_announcement = match a_events[0] {
+       assert!(a_events.len() >= 2);
+
+       // ann should be re-generated by broadcast_node_announcement - check that we have it.
+       let mut found_ann_1 = false;
+       for event in a_events.iter() {
+               match event {
+                       MessageSendEvent::BroadcastChannelAnnouncement { ref msg, .. } => {
+                               if msg == ann { found_ann_1 = true; }
+                       },
+                       MessageSendEvent::BroadcastNodeAnnouncement { .. } => {},
+                       _ => panic!("Unexpected event {:?}", event),
+               }
+       }
+       assert!(found_ann_1);
+
+       let a_node_announcement = match a_events.last().unwrap() {
                MessageSendEvent::BroadcastNodeAnnouncement { ref msg } => {
                        (*msg).clone()
                },
@@ -548,8 +644,22 @@ pub fn update_nodes_with_chan_announce<'a, 'b, 'c, 'd>(nodes: &'a Vec<Node<'b, '
 
        nodes[b].node.broadcast_node_announcement([1, 1, 1], [1; 32], Vec::new());
        let b_events = nodes[b].node.get_and_clear_pending_msg_events();
-       assert_eq!(b_events.len(), 1);
-       let b_node_announcement = match b_events[0] {
+       assert!(b_events.len() >= 2);
+
+       // ann should be re-generated by broadcast_node_announcement - check that we have it.
+       let mut found_ann_2 = false;
+       for event in b_events.iter() {
+               match event {
+                       MessageSendEvent::BroadcastChannelAnnouncement { ref msg, .. } => {
+                               if msg == ann { found_ann_2 = true; }
+                       },
+                       MessageSendEvent::BroadcastNodeAnnouncement { .. } => {},
+                       _ => panic!("Unexpected event"),
+               }
+       }
+       assert!(found_ann_2);
+
+       let b_node_announcement = match b_events.last().unwrap() {
                MessageSendEvent::BroadcastNodeAnnouncement { ref msg } => {
                        (*msg).clone()
                },
@@ -819,16 +929,29 @@ macro_rules! commitment_signed_dance {
 /// Get a payment preimage and hash.
 #[macro_export]
 macro_rules! get_payment_preimage_hash {
-       ($node: expr) => {
+       ($dest_node: expr) => {
                {
-                       let payment_preimage = PaymentPreimage([*$node.network_payment_count.borrow(); 32]);
-                       *$node.network_payment_count.borrow_mut() += 1;
+                       let payment_preimage = PaymentPreimage([*$dest_node.network_payment_count.borrow(); 32]);
+                       *$dest_node.network_payment_count.borrow_mut() += 1;
                        let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
-                       (payment_preimage, payment_hash)
+                       let payment_secret = $dest_node.node.create_inbound_payment_for_hash(payment_hash, None, 7200, 0).unwrap();
+                       (payment_preimage, payment_hash, payment_secret)
                }
        }
 }
 
+#[cfg(test)]
+macro_rules! get_route_and_payment_hash {
+       ($send_node: expr, $recv_node: expr, $recv_value: expr) => {{
+               let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!($recv_node);
+               let net_graph_msg_handler = &$send_node.net_graph_msg_handler;
+               let route = get_route(&$send_node.node.get_our_node_id(),
+                       &net_graph_msg_handler.network_graph.read().unwrap(),
+                       &$recv_node.node.get_our_node_id(), None, None, &Vec::new(), $recv_value, TEST_FINAL_CLTV, $send_node.logger).unwrap();
+               (route, payment_hash, payment_preimage, payment_secret)
+       }}
+}
+
 macro_rules! expect_pending_htlcs_forwardable_ignore {
        ($node: expr) => {{
                let events = $node.node.get_and_clear_pending_events();
@@ -847,15 +970,16 @@ macro_rules! expect_pending_htlcs_forwardable {
        }}
 }
 
-#[cfg(test)]
+#[cfg(any(test, feature = "unstable"))]
 macro_rules! expect_payment_received {
-       ($node: expr, $expected_payment_hash: expr, $expected_recv_value: expr) => {
+       ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
+                       Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
                                assert_eq!($expected_payment_hash, *payment_hash);
-                               assert_eq!(None, *payment_secret);
+                               assert!(payment_preimage.is_none());
+                               assert_eq!($expected_payment_secret, *payment_secret);
                                assert_eq!($expected_recv_value, amt);
                        },
                        _ => panic!("Unexpected event"),
@@ -876,6 +1000,30 @@ macro_rules! expect_payment_sent {
        }
 }
 
+#[cfg(test)]
+macro_rules! expect_payment_failure_chan_update {
+       ($node: expr, $scid: expr, $chan_closed: expr) => {
+               let events = $node.node.get_and_clear_pending_msg_events();
+               assert_eq!(events.len(), 1);
+               match events[0] {
+                       MessageSendEvent::PaymentFailureNetworkUpdate { ref update } => {
+                               match update {
+                                       &HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg } if !$chan_closed => {
+                                               assert_eq!(msg.contents.short_channel_id, $scid);
+                                               assert_eq!(msg.contents.flags & 2, 0);
+                                       },
+                                       &HTLCFailChannelUpdate::ChannelClosed { short_channel_id, is_permanent } if $chan_closed => {
+                                               assert_eq!(short_channel_id, $scid);
+                                               assert!(is_permanent);
+                                       },
+                                       _ => panic!("Unexpected update type"),
+                               }
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+       }
+}
+
 #[cfg(test)]
 macro_rules! expect_payment_failed {
        ($node: expr, $expected_payment_hash: expr, $rejected_by_dest: expr $(, $expected_error_code: expr, $expected_error_data: expr)*) => {
@@ -897,13 +1045,13 @@ macro_rules! expect_payment_failed {
        }
 }
 
-pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>) {
-       origin_node.node.send_payment(&route, our_payment_hash, &our_payment_secret).unwrap();
+pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
+       origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(origin_node, expected_paths.len());
        pass_along_route(origin_node, expected_paths, recv_value, our_payment_hash, our_payment_secret);
 }
 
-pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool) {
+pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret, ev: MessageSendEvent, payment_received_expected: bool) {
        let mut payment_event = SendEvent::from_event(ev);
        let mut prev_node = origin_node;
 
@@ -921,8 +1069,9 @@ pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path
                        if payment_received_expected {
                                assert_eq!(events_2.len(), 1);
                                match events_2[0] {
-                                       Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
+                                       Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
                                                assert_eq!(our_payment_hash, *payment_hash);
+                                               assert!(payment_preimage.is_none());
                                                assert_eq!(our_payment_secret, *payment_secret);
                                                assert_eq!(amt, recv_value);
                                        },
@@ -943,7 +1092,7 @@ pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path
        }
 }
 
-pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>) {
+pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
        let mut events = origin_node.node.get_and_clear_pending_msg_events();
        assert_eq!(events.len(), expected_route.len());
        for (path_idx, (ev, expected_path)) in events.drain(..).zip(expected_route.iter()).enumerate() {
@@ -954,21 +1103,17 @@ pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_rou
        }
 }
 
-pub fn send_along_route_with_hash<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash) {
-       send_along_route_with_secret(origin_node, route, &[expected_route], recv_value, our_payment_hash, None);
+pub fn send_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) {
+       let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(expected_route.last().unwrap());
+       send_along_route_with_secret(origin_node, route, &[expected_route], recv_value, our_payment_hash, our_payment_secret);
+       (our_payment_preimage, our_payment_hash, our_payment_secret)
 }
 
-pub fn send_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash) {
-       let (our_payment_preimage, our_payment_hash) = get_payment_preimage_hash!(origin_node);
-       send_along_route_with_hash(origin_node, route, expected_route, recv_value, our_payment_hash);
-       (our_payment_preimage, our_payment_hash)
-}
-
-pub fn claim_payment_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_preimage: PaymentPreimage, our_payment_secret: Option<PaymentSecret>, expected_amount: u64) {
+pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_preimage: PaymentPreimage) {
        for path in expected_paths.iter() {
                assert_eq!(path.last().unwrap().node.get_our_node_id(), expected_paths[0].last().unwrap().node.get_our_node_id());
        }
-       assert!(expected_paths[0].last().unwrap().node.claim_funds(our_payment_preimage, &our_payment_secret, expected_amount));
+       assert!(expected_paths[0].last().unwrap().node.claim_funds(our_payment_preimage));
        check_added_monitors!(expected_paths[0].last().unwrap(), expected_paths.len());
 
        macro_rules! msgs_from_ev {
@@ -1052,20 +1197,16 @@ pub fn claim_payment_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a,
        }
 }
 
-pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], skip_last: bool, our_payment_preimage: PaymentPreimage, expected_amount: u64) {
-       claim_payment_along_route_with_secret(origin_node, &[expected_route], skip_last, our_payment_preimage, None, expected_amount);
-}
-
-pub fn claim_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_preimage: PaymentPreimage, expected_amount: u64) {
-       claim_payment_along_route(origin_node, expected_route, false, our_payment_preimage, expected_amount);
+pub fn claim_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_preimage: PaymentPreimage) {
+       claim_payment_along_route(origin_node, &[expected_route], false, our_payment_preimage);
 }
 
-pub const TEST_FINAL_CLTV: u32 = 50;
+pub const TEST_FINAL_CLTV: u32 = 70;
 
-pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash) {
+pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) {
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
        let logger = test_utils::TestLogger::new();
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), None, None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
        assert_eq!(route.paths.len(), 1);
        assert_eq!(route.paths[0].len(), expected_route.len());
        for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
@@ -1078,25 +1219,25 @@ pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route:
 pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64)  {
        let logger = test_utils::TestLogger::new();
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), None, None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
+       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
        assert_eq!(route.paths.len(), 1);
        assert_eq!(route.paths[0].len(), expected_route.len());
        for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
                assert_eq!(hop.pubkey, node.node.get_our_node_id());
        }
 
-       let (_, our_payment_hash) = get_payment_preimage_hash!(origin_node);
-       unwrap_send_err!(origin_node.node.send_payment(&route, our_payment_hash, &None), true, APIError::ChannelUnavailable { ref err },
+       let (_, our_payment_hash, our_payment_preimage) = get_payment_preimage_hash!(expected_route.last().unwrap());
+       unwrap_send_err!(origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_preimage)), true, APIError::ChannelUnavailable { ref err },
                assert!(err.contains("Cannot send value that would put us over the max HTLC value in flight our peer will accept")));
 }
 
-pub fn send_payment<'a, 'b, 'c>(origin: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64, expected_value: u64)  {
+pub fn send_payment<'a, 'b, 'c>(origin: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64)  {
        let our_payment_preimage = route_payment(&origin, expected_route, recv_value).0;
-       claim_payment(&origin, expected_route, our_payment_preimage, expected_value);
+       claim_payment(&origin, expected_route, our_payment_preimage);
 }
 
 pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], skip_last: bool, our_payment_hash: PaymentHash)  {
-       assert!(expected_route.last().unwrap().node.fail_htlc_backwards(&our_payment_hash, &None));
+       assert!(expected_route.last().unwrap().node.fail_htlc_backwards(&our_payment_hash));
        expect_pending_htlcs_forwardable!(expected_route.last().unwrap());
        check_added_monitors!(expected_route.last().unwrap(), 1);
 
@@ -1171,7 +1312,10 @@ pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route:
 pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
        let mut chan_mon_cfgs = Vec::new();
        for i in 0..node_count {
-               let tx_broadcaster = test_utils::TestBroadcaster{txn_broadcasted: Mutex::new(Vec::new())};
+               let tx_broadcaster = test_utils::TestBroadcaster {
+                       txn_broadcasted: Mutex::new(Vec::new()),
+                       blocks: Arc::new(Mutex::new(vec![(genesis_block(Network::Testnet).header, 0)])),
+               };
                let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: 253 };
                let chain_source = test_utils::TestChainSource::new(Network::Testnet);
                let logger = test_utils::TestLogger::with_id(format!("node {}", i));
@@ -1210,8 +1354,7 @@ pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>
                let network = Network::Testnet;
                let params = ChainParameters {
                        network,
-                       latest_hash: genesis_block(network).header.block_hash(),
-                       latest_height: 0,
+                       best_block: BestBlock::from_genesis(network),
                };
                let node = ChannelManager::new(cfgs[i].fee_estimator, &cfgs[i].chain_monitor, cfgs[i].tx_broadcaster, cfgs[i].logger, cfgs[i].keys_manager, if node_config[i].is_some() { node_config[i].clone().unwrap() } else { default_config }, params);
                chanmgrs.push(node);
@@ -1224,6 +1367,7 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
        let mut nodes = Vec::new();
        let chan_count = Rc::new(RefCell::new(0));
        let payment_count = Rc::new(RefCell::new(0));
+       let connect_style = Rc::new(RefCell::new(ConnectStyle::FullBlockViaListen));
 
        for i in 0..node_count {
                let net_graph_msg_handler = NetGraphMsgHandler::new(cfgs[i].chain_source.genesis_hash, None, cfgs[i].logger);
@@ -1232,7 +1376,8 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
                                 keys_manager: &cfgs[i].keys_manager, node: &chan_mgrs[i], net_graph_msg_handler,
                                 node_seed: cfgs[i].node_seed, network_chan_count: chan_count.clone(),
                                 network_payment_count: payment_count.clone(), logger: cfgs[i].logger,
-                                blocks: RefCell::new(vec![(genesis_block(Network::Testnet).header, 0)])
+                                blocks: Arc::clone(&cfgs[i].tx_broadcaster.blocks),
+                                connect_style: Rc::clone(&connect_style),
                })
        }
 
@@ -1343,24 +1488,43 @@ pub fn check_preimage_claim<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, prev_txn: &Vec<
        res
 }
 
-pub fn get_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b, 'c>>, a: usize, b: usize)  {
+pub fn handle_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b, 'c>>, a: usize, b: usize, needs_err_handle: bool, expected_error: &str)  {
        let events_1 = nodes[a].node.get_and_clear_pending_msg_events();
-       assert_eq!(events_1.len(), 1);
+       assert_eq!(events_1.len(), 2);
        let as_update = match events_1[0] {
                MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                        msg.clone()
                },
                _ => panic!("Unexpected event"),
        };
+       match events_1[1] {
+               MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
+                       assert_eq!(node_id, nodes[b].node.get_our_node_id());
+                       assert_eq!(msg.data, expected_error);
+                       if needs_err_handle {
+                               nodes[b].node.handle_error(&nodes[a].node.get_our_node_id(), msg);
+                       }
+               },
+               _ => panic!("Unexpected event"),
+       }
 
        let events_2 = nodes[b].node.get_and_clear_pending_msg_events();
-       assert_eq!(events_2.len(), 1);
+       assert_eq!(events_2.len(), if needs_err_handle { 1 } else { 2 });
        let bs_update = match events_2[0] {
                MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                        msg.clone()
                },
                _ => panic!("Unexpected event"),
        };
+       if !needs_err_handle {
+               match events_2[1] {
+                       MessageSendEvent::HandleError { node_id, action: msgs::ErrorAction::SendErrorMessage { ref msg } } => {
+                               assert_eq!(node_id, nodes[a].node.get_our_node_id());
+                               assert_eq!(msg.data, expected_error);
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+       }
 
        for node in nodes {
                node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
@@ -1368,6 +1532,10 @@ pub fn get_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b,
        }
 }
 
+pub fn get_announce_close_broadcast_events<'a, 'b, 'c>(nodes: &Vec<Node<'a, 'b, 'c>>, a: usize, b: usize)  {
+       handle_announce_close_broadcast_events(nodes, a, b, false, "Commitment or closing transaction was confirmed on chain.");
+}
+
 #[cfg(test)]
 macro_rules! get_channel_value_stat {
        ($node: expr, $channel_id: expr) => {{
@@ -1407,21 +1575,28 @@ macro_rules! handle_chan_reestablish_msgs {
                                None
                        };
 
+                       if let Some(&MessageSendEvent::SendAnnouncementSignatures { ref node_id, msg: _ }) = msg_events.get(idx) {
+                               idx += 1;
+                               assert_eq!(*node_id, $dst_node.node.get_our_node_id());
+                       }
+
                        let mut revoke_and_ack = None;
                        let mut commitment_update = None;
                        let order = if let Some(ev) = msg_events.get(idx) {
-                               idx += 1;
                                match ev {
                                        &MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
                                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                                revoke_and_ack = Some(msg.clone());
+                                               idx += 1;
                                                RAACommitmentOrder::RevokeAndACKFirst
                                        },
                                        &MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
                                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                                commitment_update = Some(updates.clone());
+                                               idx += 1;
                                                RAACommitmentOrder::CommitmentFirst
                                        },
+                                       &MessageSendEvent::SendChannelUpdate { .. } => RAACommitmentOrder::CommitmentFirst,
                                        _ => panic!("Unexpected event"),
                                }
                        } else {
@@ -1434,16 +1609,24 @@ macro_rules! handle_chan_reestablish_msgs {
                                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                                assert!(revoke_and_ack.is_none());
                                                revoke_and_ack = Some(msg.clone());
+                                               idx += 1;
                                        },
                                        &MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
                                                assert_eq!(*node_id, $dst_node.node.get_our_node_id());
                                                assert!(commitment_update.is_none());
                                                commitment_update = Some(updates.clone());
+                                               idx += 1;
                                        },
+                                       &MessageSendEvent::SendChannelUpdate { .. } => {},
                                        _ => panic!("Unexpected event"),
                                }
                        }
 
+                       if let Some(&MessageSendEvent::SendChannelUpdate { ref node_id, ref msg }) = msg_events.get(idx) {
+                               assert_eq!(*node_id, $dst_node.node.get_our_node_id());
+                               assert_eq!(msg.contents.flags & 2, 0); // "disabled" flag must not be set as we just reconnected.
+                       }
+
                        (funding_locked, revoke_and_ack, commitment_update, order)
                }
        }