Penalize failed channels in Scorer
[rust-lightning] / lightning / src / ln / functional_test_utils.rs
index 2e9439916df7b8c64760813d94685621183dba01..03fa562555e362f168f8aa82044bd32c88544d31 100644 (file)
@@ -14,16 +14,17 @@ use chain::{BestBlock, Confirm, Listen, Watch};
 use chain::channelmonitor::ChannelMonitor;
 use chain::transaction::OutPoint;
 use ln::{PaymentPreimage, PaymentHash, PaymentSecret};
-use ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure};
-use routing::router::{Route, get_route};
+use ln::channelmanager::{ChainParameters, ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure, PaymentId};
 use routing::network_graph::{NetGraphMsgHandler, NetworkGraph};
+use routing::router::{Payee, Route, get_route};
+use routing::scorer::Scorer;
 use ln::features::{InitFeatures, InvoiceFeatures};
 use ln::msgs;
 use ln::msgs::{ChannelMessageHandler,RoutingMessageHandler};
 use util::enforcing_trait_impls::EnforcingSigner;
 use util::test_utils;
 use util::test_utils::TestChainMonitor;
-use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
+use util::events::{Event, MessageSendEvent, MessageSendEventsProvider, PaymentPurpose};
 use util::errors::APIError;
 use util::config::UserConfig;
 use util::ser::{ReadableArgs, Writeable, Readable};
@@ -39,10 +40,11 @@ use bitcoin::hash_types::BlockHash;
 
 use bitcoin::secp256k1::key::PublicKey;
 
+use io;
 use prelude::*;
 use core::cell::RefCell;
 use std::rc::Rc;
-use std::sync::{Arc, Mutex};
+use sync::{Arc, Mutex};
 use core::mem;
 
 pub const CHAN_CONFIRM_DEPTH: u32 = 10;
@@ -198,6 +200,7 @@ pub struct NodeCfg<'a> {
        pub keys_manager: &'a test_utils::TestKeysInterface,
        pub logger: &'a test_utils::TestLogger,
        pub node_seed: [u8; 32],
+       pub features: InitFeatures,
 }
 
 pub struct Node<'a, 'b: 'a, 'c: 'b> {
@@ -237,12 +240,12 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                        // Check that if we serialize the Router, we can deserialize it again.
                        {
                                let mut w = test_utils::TestVecWriter(Vec::new());
-                               let network_graph_ser = self.net_graph_msg_handler.network_graph.read().unwrap();
+                               let network_graph_ser = &self.net_graph_msg_handler.network_graph;
                                network_graph_ser.write(&mut w).unwrap();
-                               let network_graph_deser = <NetworkGraph>::read(&mut ::std::io::Cursor::new(&w.0)).unwrap();
-                               assert!(network_graph_deser == *self.net_graph_msg_handler.network_graph.read().unwrap());
-                               let net_graph_msg_handler = NetGraphMsgHandler::from_net_graph(
-                                       Some(self.chain_source), self.logger, network_graph_deser
+                               let network_graph_deser = <NetworkGraph>::read(&mut io::Cursor::new(&w.0)).unwrap();
+                               assert!(network_graph_deser == self.net_graph_msg_handler.network_graph);
+                               let net_graph_msg_handler = NetGraphMsgHandler::new(
+                                       network_graph_deser, Some(self.chain_source), self.logger
                                );
                                let mut chan_progress = 0;
                                loop {
@@ -269,15 +272,14 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
                        // Check that if we serialize and then deserialize all our channel monitors we get the
                        // same set of outputs to watch for on chain as we have now. Note that if we write
                        // tests that fully close channels and remove the monitors at some point this may break.
-                       let feeest = test_utils::TestFeeEstimator { sat_per_kw: 253 };
+                       let feeest = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
                        let mut deserialized_monitors = Vec::new();
                        {
-                               let old_monitors = self.chain_monitor.chain_monitor.monitors.read().unwrap();
-                               for (_, old_monitor) in old_monitors.iter() {
+                               for outpoint in self.chain_monitor.chain_monitor.list_monitors() {
                                        let mut w = test_utils::TestVecWriter(Vec::new());
-                                       old_monitor.write(&mut w).unwrap();
+                                       self.chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut w).unwrap();
                                        let (_, deserialized_monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
-                                               &mut ::std::io::Cursor::new(&w.0), self.keys_manager).unwrap();
+                                               &mut io::Cursor::new(&w.0), self.keys_manager).unwrap();
                                        deserialized_monitors.push(deserialized_monitor);
                                }
                        }
@@ -292,16 +294,16 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> {
 
                                let mut w = test_utils::TestVecWriter(Vec::new());
                                self.node.write(&mut w).unwrap();
-                               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut ::std::io::Cursor::new(w.0), ChannelManagerReadArgs {
+                               <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut io::Cursor::new(w.0), ChannelManagerReadArgs {
                                        default_config: *self.node.get_current_default_configuration(),
                                        keys_manager: self.keys_manager,
-                                       fee_estimator: &test_utils::TestFeeEstimator { sat_per_kw: 253 },
+                                       fee_estimator: &test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) },
                                        chain_monitor: self.chain_monitor,
                                        tx_broadcaster: &test_utils::TestBroadcaster {
                                                txn_broadcasted: Mutex::new(self.tx_broadcaster.txn_broadcasted.lock().unwrap().clone()),
                                                blocks: Arc::new(Mutex::new(self.tx_broadcaster.blocks.lock().unwrap().clone())),
                                        },
-                                       logger: &test_utils::TestLogger::new(),
+                                       logger: &self.logger,
                                        channel_monitors,
                                }).unwrap();
                        }
@@ -414,31 +416,56 @@ macro_rules! get_htlc_update_msgs {
        }
 }
 
+#[cfg(test)]
+macro_rules! get_channel_ref {
+       ($node: expr, $lock: ident, $channel_id: expr) => {
+               {
+                       $lock = $node.node.channel_state.lock().unwrap();
+                       $lock.by_id.get_mut(&$channel_id).unwrap()
+               }
+       }
+}
+
 #[cfg(test)]
 macro_rules! get_feerate {
        ($node: expr, $channel_id: expr) => {
                {
-                       let chan_lock = $node.node.channel_state.lock().unwrap();
-                       let chan = chan_lock.by_id.get(&$channel_id).unwrap();
+                       let mut lock;
+                       let chan = get_channel_ref!($node, lock, $channel_id);
                        chan.get_feerate()
                }
        }
 }
 
-/// Returns any local commitment transactions for the channel.
+/// Returns a channel monitor given a channel id, making some naive assumptions
 #[macro_export]
-macro_rules! get_local_commitment_txn {
+macro_rules! get_monitor {
        ($node: expr, $channel_id: expr) => {
                {
-                       let monitors = $node.chain_monitor.chain_monitor.monitors.read().unwrap();
-                       let mut commitment_txn = None;
-                       for (funding_txo, monitor) in monitors.iter() {
-                               if funding_txo.to_channel_id() == $channel_id {
-                                       commitment_txn = Some(monitor.unsafe_get_latest_holder_commitment_txn(&$node.logger));
+                       use bitcoin::hashes::Hash;
+                       let mut monitor = None;
+                       // Assume funding vout is either 0 or 1 blindly
+                       for index in 0..2 {
+                               if let Ok(mon) = $node.chain_monitor.chain_monitor.get_monitor(
+                                       $crate::chain::transaction::OutPoint {
+                                               txid: bitcoin::Txid::from_slice(&$channel_id[..]).unwrap(), index
+                                       })
+                               {
+                                       monitor = Some(mon);
                                        break;
                                }
                        }
-                       commitment_txn.unwrap()
+                       monitor.unwrap()
+               }
+       }
+}
+
+/// Returns any local commitment transactions for the channel.
+#[macro_export]
+macro_rules! get_local_commitment_txn {
+       ($node: expr, $channel_id: expr) => {
+               {
+                       $crate::get_monitor!($node, $channel_id).unsafe_get_latest_holder_commitment_txn(&$node.logger)
                }
        }
 }
@@ -500,11 +527,12 @@ pub fn create_funding_transaction<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, expected_
 }
 
 pub fn create_chan_between_nodes_with_value_init<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a, 'b, 'c>, channel_value: u64, push_msat: u64, a_flags: InitFeatures, b_flags: InitFeatures) -> Transaction {
-       node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
+       let create_chan_id = node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
        node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), a_flags, &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
        node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), b_flags, &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
 
        let (temporary_channel_id, tx, funding_output) = create_funding_transaction(node_a, channel_value, 42);
+       assert_eq!(temporary_channel_id, create_chan_id);
 
        node_a.node.funding_transaction_generated(&temporary_channel_id, tx.clone()).unwrap();
        check_added_monitors!(node_a, 0);
@@ -731,16 +759,16 @@ macro_rules! get_closing_signed_broadcast {
 #[macro_export]
 macro_rules! check_closed_broadcast {
        ($node: expr, $with_error_msg: expr) => {{
-               let events = $node.node.get_and_clear_pending_msg_events();
-               assert_eq!(events.len(), if $with_error_msg { 2 } else { 1 });
-               match events[0] {
+               let msg_events = $node.node.get_and_clear_pending_msg_events();
+               assert_eq!(msg_events.len(), if $with_error_msg { 2 } else { 1 });
+               match msg_events[0] {
                        MessageSendEvent::BroadcastChannelUpdate { ref msg } => {
                                assert_eq!(msg.contents.flags & 2, 2);
                        },
                        _ => panic!("Unexpected event"),
                }
                if $with_error_msg {
-                       match events[1] {
+                       match msg_events[1] {
                                MessageSendEvent::HandleError { action: ErrorAction::SendErrorMessage { ref msg }, node_id: _ } => {
                                        // TODO: Check node_id
                                        Some(msg.clone())
@@ -751,9 +779,35 @@ macro_rules! check_closed_broadcast {
        }}
 }
 
+/// Check that a channel's closing channel events has been issued
+#[macro_export]
+macro_rules! check_closed_event {
+       ($node: expr, $events: expr, $reason: expr) => {
+               check_closed_event!($node, $events, $reason, false);
+       };
+       ($node: expr, $events: expr, $reason: expr, $is_check_discard_funding: expr) => {{
+               let events = $node.node.get_and_clear_pending_events();
+               assert_eq!(events.len(), $events);
+               let expected_reason = $reason;
+               let mut issues_discard_funding = false;
+               for event in events {
+                       match event {
+                               Event::ChannelClosed { ref reason, .. } => {
+                                       assert_eq!(*reason, expected_reason);
+                               },
+                               Event::DiscardFunding { .. } => {
+                                       issues_discard_funding = true;
+                               }
+                               _ => panic!("Unexpected event"),
+                       }
+               }
+               assert_eq!($is_check_discard_funding, issues_discard_funding);
+       }}
+}
+
 pub fn close_channel<'a, 'b, 'c>(outbound_node: &Node<'a, 'b, 'c>, inbound_node: &Node<'a, 'b, 'c>, channel_id: &[u8; 32], funding_tx: Transaction, close_inbound_first: bool) -> (msgs::ChannelUpdate, msgs::ChannelUpdate, Transaction) {
        let (node_a, broadcaster_a, struct_a) = if close_inbound_first { (&inbound_node.node, &inbound_node.tx_broadcaster, inbound_node) } else { (&outbound_node.node, &outbound_node.tx_broadcaster, outbound_node) };
-       let (node_b, broadcaster_b) = if close_inbound_first { (&outbound_node.node, &outbound_node.tx_broadcaster) } else { (&inbound_node.node, &inbound_node.tx_broadcaster) };
+       let (node_b, broadcaster_b, struct_b) = if close_inbound_first { (&outbound_node.node, &outbound_node.tx_broadcaster, outbound_node) } else { (&inbound_node.node, &inbound_node.tx_broadcaster, inbound_node) };
        let (tx_a, tx_b);
 
        node_a.close_channel(channel_id).unwrap();
@@ -786,29 +840,33 @@ pub fn close_channel<'a, 'b, 'c>(outbound_node: &Node<'a, 'b, 'c>, inbound_node:
        let (as_update, bs_update) = if close_inbound_first {
                assert!(node_a.get_and_clear_pending_msg_events().is_empty());
                node_a.handle_closing_signed(&node_b.get_our_node_id(), &closing_signed_b.unwrap());
-               assert_eq!(broadcaster_a.txn_broadcasted.lock().unwrap().len(), 1);
-               tx_a = broadcaster_a.txn_broadcasted.lock().unwrap().remove(0);
-               let (as_update, closing_signed_a) = get_closing_signed_broadcast!(node_a, node_b.get_our_node_id());
 
-               node_b.handle_closing_signed(&node_a.get_our_node_id(), &closing_signed_a.unwrap());
-               let (bs_update, none_b) = get_closing_signed_broadcast!(node_b, node_a.get_our_node_id());
-               assert!(none_b.is_none());
+               node_b.handle_closing_signed(&node_a.get_our_node_id(), &get_event_msg!(struct_a, MessageSendEvent::SendClosingSigned, node_b.get_our_node_id()));
                assert_eq!(broadcaster_b.txn_broadcasted.lock().unwrap().len(), 1);
                tx_b = broadcaster_b.txn_broadcasted.lock().unwrap().remove(0);
+               let (bs_update, closing_signed_b) = get_closing_signed_broadcast!(node_b, node_a.get_our_node_id());
+
+               node_a.handle_closing_signed(&node_b.get_our_node_id(), &closing_signed_b.unwrap());
+               let (as_update, none_a) = get_closing_signed_broadcast!(node_a, node_b.get_our_node_id());
+               assert!(none_a.is_none());
+               assert_eq!(broadcaster_a.txn_broadcasted.lock().unwrap().len(), 1);
+               tx_a = broadcaster_a.txn_broadcasted.lock().unwrap().remove(0);
                (as_update, bs_update)
        } else {
                let closing_signed_a = get_event_msg!(struct_a, MessageSendEvent::SendClosingSigned, node_b.get_our_node_id());
 
                node_b.handle_closing_signed(&node_a.get_our_node_id(), &closing_signed_a);
-               assert_eq!(broadcaster_b.txn_broadcasted.lock().unwrap().len(), 1);
-               tx_b = broadcaster_b.txn_broadcasted.lock().unwrap().remove(0);
-               let (bs_update, closing_signed_b) = get_closing_signed_broadcast!(node_b, node_a.get_our_node_id());
+               node_a.handle_closing_signed(&node_b.get_our_node_id(), &get_event_msg!(struct_b, MessageSendEvent::SendClosingSigned, node_a.get_our_node_id()));
 
-               node_a.handle_closing_signed(&node_b.get_our_node_id(), &closing_signed_b.unwrap());
-               let (as_update, none_a) = get_closing_signed_broadcast!(node_a, node_b.get_our_node_id());
-               assert!(none_a.is_none());
                assert_eq!(broadcaster_a.txn_broadcasted.lock().unwrap().len(), 1);
                tx_a = broadcaster_a.txn_broadcasted.lock().unwrap().remove(0);
+               let (as_update, closing_signed_a) = get_closing_signed_broadcast!(node_a, node_b.get_our_node_id());
+
+               node_b.handle_closing_signed(&node_a.get_our_node_id(), &closing_signed_a.unwrap());
+               let (bs_update, none_b) = get_closing_signed_broadcast!(node_b, node_a.get_our_node_id());
+               assert!(none_b.is_none());
+               assert_eq!(broadcaster_b.txn_broadcasted.lock().unwrap().len(), 1);
+               tx_b = broadcaster_b.txn_broadcasted.lock().unwrap().remove(0);
                (as_update, bs_update)
        };
        assert_eq!(tx_a, tx_b);
@@ -931,10 +989,16 @@ macro_rules! commitment_signed_dance {
 macro_rules! get_payment_preimage_hash {
        ($dest_node: expr) => {
                {
-                       let payment_preimage = PaymentPreimage([*$dest_node.network_payment_count.borrow(); 32]);
-                       *$dest_node.network_payment_count.borrow_mut() += 1;
+                       get_payment_preimage_hash!($dest_node, None)
+               }
+       };
+       ($dest_node: expr, $min_value_msat: expr) => {
+               {
+                       let mut payment_count = $dest_node.network_payment_count.borrow_mut();
+                       let payment_preimage = PaymentPreimage([*payment_count; 32]);
+                       *payment_count += 1;
                        let payment_hash = PaymentHash(Sha256::hash(&payment_preimage.0[..]).into_inner());
-                       let payment_secret = $dest_node.node.create_inbound_payment_for_hash(payment_hash, None, 7200, 0).unwrap();
+                       let payment_secret = $dest_node.node.create_inbound_payment_for_hash(payment_hash, $min_value_msat, 7200, 0).unwrap();
                        (payment_preimage, payment_hash, payment_secret)
                }
        }
@@ -943,11 +1007,20 @@ macro_rules! get_payment_preimage_hash {
 #[cfg(test)]
 macro_rules! get_route_and_payment_hash {
        ($send_node: expr, $recv_node: expr, $recv_value: expr) => {{
-               let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!($recv_node);
+               get_route_and_payment_hash!($send_node, $recv_node, vec![], $recv_value, TEST_FINAL_CLTV)
+       }};
+       ($send_node: expr, $recv_node: expr, $last_hops: expr, $recv_value: expr, $cltv: expr) => {{
+               let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!($recv_node, Some($recv_value));
+               let payee = $crate::routing::router::Payee::new($recv_node.node.get_our_node_id())
+                       .with_features($crate::ln::features::InvoiceFeatures::known())
+                       .with_route_hints($last_hops);
                let net_graph_msg_handler = &$send_node.net_graph_msg_handler;
-               let route = get_route(&$send_node.node.get_our_node_id(),
-                       &net_graph_msg_handler.network_graph.read().unwrap(),
-                       &$recv_node.node.get_our_node_id(), None, None, &Vec::new(), $recv_value, TEST_FINAL_CLTV, $send_node.logger).unwrap();
+               let scorer = ::routing::scorer::Scorer::with_fixed_penalty(0);
+               let route = ::routing::router::get_route(
+                       &$send_node.node.get_our_node_id(), &payee, &net_graph_msg_handler.network_graph,
+                       Some(&$send_node.node.list_usable_channels().iter().collect::<Vec<_>>()),
+                       $recv_value, $cltv, $send_node.logger, &scorer
+               ).unwrap();
                (route, payment_hash, payment_preimage, payment_secret)
        }}
 }
@@ -970,17 +1043,36 @@ macro_rules! expect_pending_htlcs_forwardable {
        }}
 }
 
+#[cfg(test)]
+macro_rules! expect_pending_htlcs_forwardable_from_events {
+       ($node: expr, $events: expr, $ignore: expr) => {{
+               assert_eq!($events.len(), 1);
+               match $events[0] {
+                       Event::PendingHTLCsForwardable { .. } => { },
+                       _ => panic!("Unexpected event"),
+               };
+               if $ignore {
+                       $node.node.process_pending_htlc_forwards();
+               }
+       }}
+}
+
 #[cfg(any(test, feature = "unstable"))]
 macro_rules! expect_payment_received {
        ($node: expr, $expected_payment_hash: expr, $expected_payment_secret: expr, $expected_recv_value: expr) => {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
+                       Event::PaymentReceived { ref payment_hash, ref purpose, amt } => {
                                assert_eq!($expected_payment_hash, *payment_hash);
-                               assert!(payment_preimage.is_none());
-                               assert_eq!($expected_payment_secret, *payment_secret);
                                assert_eq!($expected_recv_value, amt);
+                               match purpose {
+                                       PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
+                                               assert!(payment_preimage.is_none());
+                                               assert_eq!($expected_payment_secret, *payment_secret);
+                                       },
+                                       _ => {},
+                               }
                        },
                        _ => panic!("Unexpected event"),
                }
@@ -990,10 +1082,26 @@ macro_rules! expect_payment_received {
 macro_rules! expect_payment_sent {
        ($node: expr, $expected_payment_preimage: expr) => {
                let events = $node.node.get_and_clear_pending_events();
+               let expected_payment_hash = PaymentHash(Sha256::hash(&$expected_payment_preimage.0).into_inner());
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentSent { ref payment_preimage } => {
+                       Event::PaymentSent { payment_id: _, ref payment_preimage, ref payment_hash } => {
                                assert_eq!($expected_payment_preimage, *payment_preimage);
+                               assert_eq!(expected_payment_hash, *payment_hash);
+                       },
+                       _ => panic!("Unexpected event"),
+               }
+       }
+}
+
+macro_rules! expect_payment_forwarded {
+       ($node: expr, $expected_fee: expr, $upstream_force_closed: expr) => {
+               let events = $node.node.get_and_clear_pending_events();
+               assert_eq!(events.len(), 1);
+               match events[0] {
+                       Event::PaymentForwarded { fee_earned_msat, claim_from_onchain_tx } => {
+                               assert_eq!(fee_earned_msat, $expected_fee);
+                               assert_eq!(claim_from_onchain_tx, $upstream_force_closed);
                        },
                        _ => panic!("Unexpected event"),
                }
@@ -1001,22 +1109,30 @@ macro_rules! expect_payment_sent {
 }
 
 #[cfg(test)]
-macro_rules! expect_payment_failure_chan_update {
-       ($node: expr, $scid: expr, $chan_closed: expr) => {
-               let events = $node.node.get_and_clear_pending_msg_events();
+macro_rules! expect_payment_failed_with_update {
+       ($node: expr, $expected_payment_hash: expr, $rejected_by_dest: expr, $scid: expr, $chan_closed: expr) => {
+               let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       MessageSendEvent::PaymentFailureNetworkUpdate { ref update } => {
-                               match update {
-                                       &HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg } if !$chan_closed => {
+                       Event::PaymentPathFailed { ref payment_hash, rejected_by_dest, ref network_update, ref error_code, ref error_data, ref path, ref retry, .. } => {
+                               assert_eq!(*payment_hash, $expected_payment_hash, "unexpected payment_hash");
+                               assert_eq!(rejected_by_dest, $rejected_by_dest, "unexpected rejected_by_dest value");
+                               assert!(retry.is_some(), "expected retry.is_some()");
+                               assert_eq!(retry.as_ref().unwrap().final_value_msat, path.last().unwrap().fee_msat, "Retry amount should match last hop in path");
+                               assert_eq!(retry.as_ref().unwrap().payee.pubkey, path.last().unwrap().pubkey, "Retry payee node_id should match last hop in path");
+                               assert!(error_code.is_some(), "expected error_code.is_some() = true");
+                               assert!(error_data.is_some(), "expected error_data.is_some() = true");
+                               match network_update {
+                                       &Some(NetworkUpdate::ChannelUpdateMessage { ref msg }) if !$chan_closed => {
                                                assert_eq!(msg.contents.short_channel_id, $scid);
                                                assert_eq!(msg.contents.flags & 2, 0);
                                        },
-                                       &HTLCFailChannelUpdate::ChannelClosed { short_channel_id, is_permanent } if $chan_closed => {
+                                       &Some(NetworkUpdate::ChannelClosed { short_channel_id, is_permanent }) if $chan_closed => {
                                                assert_eq!(short_channel_id, $scid);
                                                assert!(is_permanent);
                                        },
-                                       _ => panic!("Unexpected update type"),
+                                       Some(_) => panic!("Unexpected update type"),
+                                       None => panic!("Expected update"),
                                }
                        },
                        _ => panic!("Unexpected event"),
@@ -1030,9 +1146,12 @@ macro_rules! expect_payment_failed {
                let events = $node.node.get_and_clear_pending_events();
                assert_eq!(events.len(), 1);
                match events[0] {
-                       Event::PaymentFailed { ref payment_hash, rejected_by_dest, ref error_code, ref error_data } => {
+                       Event::PaymentPathFailed { ref payment_hash, rejected_by_dest, network_update: _, ref error_code, ref error_data, ref path, ref retry, .. } => {
                                assert_eq!(*payment_hash, $expected_payment_hash, "unexpected payment_hash");
                                assert_eq!(rejected_by_dest, $rejected_by_dest, "unexpected rejected_by_dest value");
+                               assert!(retry.is_some(), "expected retry.is_some()");
+                               assert_eq!(retry.as_ref().unwrap().final_value_msat, path.last().unwrap().fee_msat, "Retry amount should match last hop in path");
+                               assert_eq!(retry.as_ref().unwrap().payee.pubkey, path.last().unwrap().pubkey, "Retry payee node_id should match last hop in path");
                                assert!(error_code.is_some(), "expected error_code.is_some() = true");
                                assert!(error_data.is_some(), "expected error_data.is_some() = true");
                                $(
@@ -1045,13 +1164,14 @@ macro_rules! expect_payment_failed {
        }
 }
 
-pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) {
-       origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
+pub fn send_along_route_with_secret<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_paths: &[&[&Node<'a, 'b, 'c>]], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret) -> PaymentId {
+       let payment_id = origin_node.node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
        check_added_monitors!(origin_node, expected_paths.len());
        pass_along_route(origin_node, expected_paths, recv_value, our_payment_hash, our_payment_secret);
+       payment_id
 }
 
-pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: PaymentSecret, ev: MessageSendEvent, payment_received_expected: bool) {
+pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], recv_value: u64, our_payment_hash: PaymentHash, our_payment_secret: Option<PaymentSecret>, ev: MessageSendEvent, payment_received_expected: bool, expected_preimage: Option<PaymentPreimage>) {
        let mut payment_event = SendEvent::from_event(ev);
        let mut prev_node = origin_node;
 
@@ -1069,10 +1189,18 @@ pub fn pass_along_path<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path
                        if payment_received_expected {
                                assert_eq!(events_2.len(), 1);
                                match events_2[0] {
-                                       Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
+                                       Event::PaymentReceived { ref payment_hash, ref purpose, amt} => {
                                                assert_eq!(our_payment_hash, *payment_hash);
-                                               assert!(payment_preimage.is_none());
-                                               assert_eq!(our_payment_secret, *payment_secret);
+                                               match &purpose {
+                                                       PaymentPurpose::InvoicePayment { payment_preimage, payment_secret, .. } => {
+                                                               assert_eq!(expected_preimage, *payment_preimage);
+                                                               assert_eq!(our_payment_secret.unwrap(), *payment_secret);
+                                                       },
+                                                       PaymentPurpose::SpontaneousPayment(payment_preimage) => {
+                                                               assert_eq!(expected_preimage.unwrap(), *payment_preimage);
+                                                               assert!(our_payment_secret.is_none());
+                                                       },
+                                               }
                                                assert_eq!(amt, recv_value);
                                        },
                                        _ => panic!("Unexpected event"),
@@ -1099,14 +1227,14 @@ pub fn pass_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_rou
                // Once we've gotten through all the HTLCs, the last one should result in a
                // PaymentReceived (but each previous one should not!), .
                let expect_payment = path_idx == expected_route.len() - 1;
-               pass_along_path(origin_node, expected_path, recv_value, our_payment_hash.clone(), our_payment_secret, ev, expect_payment);
+               pass_along_path(origin_node, expected_path, recv_value, our_payment_hash.clone(), Some(our_payment_secret), ev, expect_payment, None);
        }
 }
 
-pub fn send_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) {
+pub fn send_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, route: Route, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret, PaymentId) {
        let (our_payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(expected_route.last().unwrap());
-       send_along_route_with_secret(origin_node, route, &[expected_route], recv_value, our_payment_hash, our_payment_secret);
-       (our_payment_preimage, our_payment_hash, our_payment_secret)
+       let payment_id = send_along_route_with_secret(origin_node, route, &[expected_route], recv_value, our_payment_hash, our_payment_secret);
+       (our_payment_preimage, our_payment_hash, our_payment_secret, payment_id)
 }
 
 pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_preimage: PaymentPreimage) {
@@ -1156,6 +1284,8 @@ pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, exp
                        ($node: expr, $prev_node: expr, $new_msgs: expr) => {
                                {
                                        $node.node.handle_update_fulfill_htlc(&$prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
+                                       let fee = $node.node.channel_state.lock().unwrap().by_id.get(&next_msgs.as_ref().unwrap().0.channel_id).unwrap().config.forwarding_fee_base_msat;
+                                       expect_payment_forwarded!($node, Some(fee as u64), false);
                                        check_added_monitors!($node, 1);
                                        let new_next_msgs = if $new_msgs {
                                                let events = $node.node.get_and_clear_pending_msg_events();
@@ -1192,9 +1322,11 @@ pub fn claim_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, exp
 
                if !skip_last {
                        last_update_fulfill_dance!(origin_node, expected_route.first().unwrap());
-                       expect_payment_sent!(origin_node, our_payment_preimage);
                }
        }
+       if !skip_last {
+               expect_payment_sent!(origin_node, our_payment_preimage);
+       }
 }
 
 pub fn claim_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_preimage: PaymentPreimage) {
@@ -1204,22 +1336,30 @@ pub fn claim_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route:
 pub const TEST_FINAL_CLTV: u32 = 70;
 
 pub fn route_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64) -> (PaymentPreimage, PaymentHash, PaymentSecret) {
+       let payee = Payee::new(expected_route.last().unwrap().node.get_our_node_id())
+               .with_features(InvoiceFeatures::known());
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
-       let logger = test_utils::TestLogger::new();
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
+       let scorer = Scorer::with_fixed_penalty(0);
+       let route = get_route(
+               &origin_node.node.get_our_node_id(), &payee, &net_graph_msg_handler.network_graph,
+               Some(&origin_node.node.list_usable_channels().iter().collect::<Vec<_>>()),
+               recv_value, TEST_FINAL_CLTV, origin_node.logger, &scorer).unwrap();
        assert_eq!(route.paths.len(), 1);
        assert_eq!(route.paths[0].len(), expected_route.len());
        for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
                assert_eq!(hop.pubkey, node.node.get_our_node_id());
        }
 
-       send_along_route(origin_node, route, expected_route, recv_value)
+       let res = send_along_route(origin_node, route, expected_route, recv_value);
+       (res.0, res.1, res.2)
 }
 
 pub fn route_over_limit<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], recv_value: u64)  {
-       let logger = test_utils::TestLogger::new();
+       let payee = Payee::new(expected_route.last().unwrap().node.get_our_node_id())
+               .with_features(InvoiceFeatures::known());
        let net_graph_msg_handler = &origin_node.net_graph_msg_handler;
-       let route = get_route(&origin_node.node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &expected_route.last().unwrap().node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), recv_value, TEST_FINAL_CLTV, &logger).unwrap();
+       let scorer = Scorer::with_fixed_penalty(0);
+       let route = get_route(&origin_node.node.get_our_node_id(), &payee, &net_graph_msg_handler.network_graph, None, recv_value, TEST_FINAL_CLTV, origin_node.logger, &scorer).unwrap();
        assert_eq!(route.paths.len(), 1);
        assert_eq!(route.paths[0].len(), expected_route.len());
        for (node, hop) in expected_route.iter().zip(route.paths[0].iter()) {
@@ -1236,77 +1376,100 @@ pub fn send_payment<'a, 'b, 'c>(origin: &Node<'a, 'b, 'c>, expected_route: &[&No
        claim_payment(&origin, expected_route, our_payment_preimage);
 }
 
-pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], skip_last: bool, our_payment_hash: PaymentHash)  {
-       assert!(expected_route.last().unwrap().node.fail_htlc_backwards(&our_payment_hash));
-       expect_pending_htlcs_forwardable!(expected_route.last().unwrap());
-       check_added_monitors!(expected_route.last().unwrap(), 1);
+pub fn fail_payment_along_route<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_paths_slice: &[&[&Node<'a, 'b, 'c>]], skip_last: bool, our_payment_hash: PaymentHash)  {
+       let mut expected_paths: Vec<_> = expected_paths_slice.iter().collect();
+       for path in expected_paths.iter() {
+               assert_eq!(path.last().unwrap().node.get_our_node_id(), expected_paths[0].last().unwrap().node.get_our_node_id());
+       }
+       assert!(expected_paths[0].last().unwrap().node.fail_htlc_backwards(&our_payment_hash));
+       expect_pending_htlcs_forwardable!(expected_paths[0].last().unwrap());
+       check_added_monitors!(expected_paths[0].last().unwrap(), expected_paths.len());
+
+       let mut per_path_msgs: Vec<((msgs::UpdateFailHTLC, msgs::CommitmentSigned), PublicKey)> = Vec::with_capacity(expected_paths.len());
+       let events = expected_paths[0].last().unwrap().node.get_and_clear_pending_msg_events();
+       assert_eq!(events.len(), expected_paths.len());
+       for ev in events.iter() {
+               let (update_fail, commitment_signed, node_id) = match ev {
+                       &MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+                               assert!(update_add_htlcs.is_empty());
+                               assert!(update_fulfill_htlcs.is_empty());
+                               assert_eq!(update_fail_htlcs.len(), 1);
+                               assert!(update_fail_malformed_htlcs.is_empty());
+                               assert!(update_fee.is_none());
+                               (update_fail_htlcs[0].clone(), commitment_signed.clone(), node_id.clone())
+                       },
+                       _ => panic!("Unexpected event"),
+               };
+               per_path_msgs.push(((update_fail, commitment_signed), node_id));
+       }
+       per_path_msgs.sort_unstable_by(|(_, node_id_a), (_, node_id_b)| node_id_a.cmp(node_id_b));
+       expected_paths.sort_unstable_by(|path_a, path_b| path_a[path_a.len() - 2].node.get_our_node_id().cmp(&path_b[path_b.len() - 2].node.get_our_node_id()));
 
-       let mut next_msgs: Option<(msgs::UpdateFailHTLC, msgs::CommitmentSigned)> = None;
-       macro_rules! update_fail_dance {
-               ($node: expr, $prev_node: expr, $last_node: expr) => {
-                       {
-                               $node.node.handle_update_fail_htlc(&$prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
-                               commitment_signed_dance!($node, $prev_node, next_msgs.as_ref().unwrap().1, !$last_node);
-                               if skip_last && $last_node {
-                                       expect_pending_htlcs_forwardable!($node);
+       for (i, (expected_route, (path_msgs, next_hop))) in expected_paths.iter().zip(per_path_msgs.drain(..)).enumerate() {
+               let mut next_msgs = Some(path_msgs);
+               let mut expected_next_node = next_hop;
+               let mut prev_node = expected_route.last().unwrap();
+
+               for (idx, node) in expected_route.iter().rev().enumerate().skip(1) {
+                       assert_eq!(expected_next_node, node.node.get_our_node_id());
+                       let update_next_node = !skip_last || idx != expected_route.len() - 1;
+                       if next_msgs.is_some() {
+                               node.node.handle_update_fail_htlc(&prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
+                               commitment_signed_dance!(node, prev_node, next_msgs.as_ref().unwrap().1, update_next_node);
+                               if !update_next_node {
+                                       expect_pending_htlcs_forwardable!(node);
                                }
                        }
-               }
-       }
+                       let events = node.node.get_and_clear_pending_msg_events();
+                       if update_next_node {
+                               assert_eq!(events.len(), 1);
+                               match events[0] {
+                                       MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
+                                               assert!(update_add_htlcs.is_empty());
+                                               assert!(update_fulfill_htlcs.is_empty());
+                                               assert_eq!(update_fail_htlcs.len(), 1);
+                                               assert!(update_fail_malformed_htlcs.is_empty());
+                                               assert!(update_fee.is_none());
+                                               expected_next_node = node_id.clone();
+                                               next_msgs = Some((update_fail_htlcs[0].clone(), commitment_signed.clone()));
+                                       },
+                                       _ => panic!("Unexpected event"),
+                               }
+                       } else {
+                               assert!(events.is_empty());
+                       }
+                       if !skip_last && idx == expected_route.len() - 1 {
+                               assert_eq!(expected_next_node, origin_node.node.get_our_node_id());
+                       }
 
-       let mut expected_next_node = expected_route.last().unwrap().node.get_our_node_id();
-       let mut prev_node = expected_route.last().unwrap();
-       for (idx, node) in expected_route.iter().rev().enumerate() {
-               assert_eq!(expected_next_node, node.node.get_our_node_id());
-               if next_msgs.is_some() {
-                       // We may be the "last node" for the purpose of the commitment dance if we're
-                       // skipping the last node (implying it is disconnected) and we're the
-                       // second-to-last node!
-                       update_fail_dance!(node, prev_node, skip_last && idx == expected_route.len() - 1);
+                       prev_node = node;
                }
 
-               let events = node.node.get_and_clear_pending_msg_events();
-               if !skip_last || idx != expected_route.len() - 1 {
+               if !skip_last {
+                       let prev_node = expected_route.first().unwrap();
+                       origin_node.node.handle_update_fail_htlc(&prev_node.node.get_our_node_id(), &next_msgs.as_ref().unwrap().0);
+                       check_added_monitors!(origin_node, 0);
+                       assert!(origin_node.node.get_and_clear_pending_msg_events().is_empty());
+                       commitment_signed_dance!(origin_node, prev_node, next_msgs.as_ref().unwrap().1, false);
+                       let events = origin_node.node.get_and_clear_pending_events();
                        assert_eq!(events.len(), 1);
                        match events[0] {
-                               MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
-                                       assert!(update_add_htlcs.is_empty());
-                                       assert!(update_fulfill_htlcs.is_empty());
-                                       assert_eq!(update_fail_htlcs.len(), 1);
-                                       assert!(update_fail_malformed_htlcs.is_empty());
-                                       assert!(update_fee.is_none());
-                                       expected_next_node = node_id.clone();
-                                       next_msgs = Some((update_fail_htlcs[0].clone(), commitment_signed.clone()));
+                               Event::PaymentPathFailed { payment_hash, rejected_by_dest, all_paths_failed, ref path, .. } => {
+                                       assert_eq!(payment_hash, our_payment_hash);
+                                       assert!(rejected_by_dest);
+                                       assert_eq!(all_paths_failed, i == expected_paths.len() - 1);
+                                       for (idx, hop) in expected_route.iter().enumerate() {
+                                               assert_eq!(hop.node.get_our_node_id(), path[idx].pubkey);
+                                       }
                                },
                                _ => panic!("Unexpected event"),
                        }
-               } else {
-                       assert!(events.is_empty());
-               }
-               if !skip_last && idx == expected_route.len() - 1 {
-                       assert_eq!(expected_next_node, origin_node.node.get_our_node_id());
-               }
-
-               prev_node = node;
-       }
-
-       if !skip_last {
-               update_fail_dance!(origin_node, expected_route.first().unwrap(), true);
-
-               let events = origin_node.node.get_and_clear_pending_events();
-               assert_eq!(events.len(), 1);
-               match events[0] {
-                       Event::PaymentFailed { payment_hash, rejected_by_dest, .. } => {
-                               assert_eq!(payment_hash, our_payment_hash);
-                               assert!(rejected_by_dest);
-                       },
-                       _ => panic!("Unexpected event"),
                }
        }
 }
 
-pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_route: &[&Node<'a, 'b, 'c>], our_payment_hash: PaymentHash)  {
-       fail_payment_along_route(origin_node, expected_route, false, our_payment_hash);
+pub fn fail_payment<'a, 'b, 'c>(origin_node: &Node<'a, 'b, 'c>, expected_path: &[&Node<'a, 'b, 'c>], our_payment_hash: PaymentHash)  {
+       fail_payment_along_route(origin_node, &[&expected_path[..]], false, our_payment_hash);
 }
 
 pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
@@ -1316,7 +1479,7 @@ pub fn create_chanmon_cfgs(node_count: usize) -> Vec<TestChanMonCfg> {
                        txn_broadcasted: Mutex::new(Vec::new()),
                        blocks: Arc::new(Mutex::new(vec![(genesis_block(Network::Testnet).header, 0)])),
                };
-               let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: 253 };
+               let fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
                let chain_source = test_utils::TestChainSource::new(Network::Testnet);
                let logger = test_utils::TestLogger::with_id(format!("node {}", i));
                let persister = test_utils::TestPersister::new();
@@ -1335,28 +1498,47 @@ pub fn create_node_cfgs<'a>(node_count: usize, chanmon_cfgs: &'a Vec<TestChanMon
        for i in 0..node_count {
                let chain_monitor = test_utils::TestChainMonitor::new(Some(&chanmon_cfgs[i].chain_source), &chanmon_cfgs[i].tx_broadcaster, &chanmon_cfgs[i].logger, &chanmon_cfgs[i].fee_estimator, &chanmon_cfgs[i].persister, &chanmon_cfgs[i].keys_manager);
                let seed = [i as u8; 32];
-               nodes.push(NodeCfg { chain_source: &chanmon_cfgs[i].chain_source, logger: &chanmon_cfgs[i].logger, tx_broadcaster: &chanmon_cfgs[i].tx_broadcaster, fee_estimator: &chanmon_cfgs[i].fee_estimator, chain_monitor, keys_manager: &chanmon_cfgs[i].keys_manager, node_seed: seed });
+               nodes.push(NodeCfg {
+                       chain_source: &chanmon_cfgs[i].chain_source,
+                       logger: &chanmon_cfgs[i].logger,
+                       tx_broadcaster: &chanmon_cfgs[i].tx_broadcaster,
+                       fee_estimator: &chanmon_cfgs[i].fee_estimator,
+                       chain_monitor,
+                       keys_manager: &chanmon_cfgs[i].keys_manager,
+                       node_seed: seed,
+                       features: InitFeatures::known(),
+               });
        }
 
        nodes
 }
 
+pub fn test_default_channel_config() -> UserConfig {
+       let mut default_config = UserConfig::default();
+       // Set cltv_expiry_delta slightly lower to keep the final CLTV values inside one byte in our
+       // tests so that our script-length checks don't fail (see ACCEPTED_HTLC_SCRIPT_WEIGHT).
+       default_config.channel_options.cltv_expiry_delta = 6*6;
+       default_config.channel_options.announced_channel = true;
+       default_config.peer_channel_config_limits.force_announced_channel_preference = false;
+       // When most of our tests were written, the default HTLC minimum was fixed at 1000.
+       // It now defaults to 1, so we simply set it to the expected value here.
+       default_config.own_channel_config.our_htlc_minimum_msat = 1000;
+       // When most of our tests were written, we didn't have the notion of a `max_dust_htlc_exposure_msat`,
+       // It now defaults to 5_000_000 msat; to avoid interfering with tests we bump it to 50_000_000 msat.
+       default_config.channel_options.max_dust_htlc_exposure_msat = 50_000_000;
+       default_config
+}
+
 pub fn create_node_chanmgrs<'a, 'b>(node_count: usize, cfgs: &'a Vec<NodeCfg<'b>>, node_config: &[Option<UserConfig>]) -> Vec<ChannelManager<EnforcingSigner, &'a TestChainMonitor<'b>, &'b test_utils::TestBroadcaster, &'a test_utils::TestKeysInterface, &'b test_utils::TestFeeEstimator, &'b test_utils::TestLogger>> {
        let mut chanmgrs = Vec::new();
        for i in 0..node_count {
-               let mut default_config = UserConfig::default();
-               // Set cltv_expiry_delta slightly lower to keep the final CLTV values inside one byte in our
-               // tests so that our script-length checks don't fail (see ACCEPTED_HTLC_SCRIPT_WEIGHT).
-               default_config.channel_options.cltv_expiry_delta = 6*6;
-               default_config.channel_options.announced_channel = true;
-               default_config.peer_channel_config_limits.force_announced_channel_preference = false;
-               default_config.own_channel_config.our_htlc_minimum_msat = 1000; // sanitization being done by the sender, to exerce receiver logic we need to lift of limit
                let network = Network::Testnet;
                let params = ChainParameters {
                        network,
                        best_block: BestBlock::from_genesis(network),
                };
-               let node = ChannelManager::new(cfgs[i].fee_estimator, &cfgs[i].chain_monitor, cfgs[i].tx_broadcaster, cfgs[i].logger, cfgs[i].keys_manager, if node_config[i].is_some() { node_config[i].clone().unwrap() } else { default_config }, params);
+               let node = ChannelManager::new(cfgs[i].fee_estimator, &cfgs[i].chain_monitor, cfgs[i].tx_broadcaster, cfgs[i].logger, cfgs[i].keys_manager,
+                       if node_config[i].is_some() { node_config[i].clone().unwrap() } else { test_default_channel_config() }, params);
                chanmgrs.push(node);
        }
 
@@ -1370,7 +1552,8 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
        let connect_style = Rc::new(RefCell::new(ConnectStyle::FullBlockViaListen));
 
        for i in 0..node_count {
-               let net_graph_msg_handler = NetGraphMsgHandler::new(cfgs[i].chain_source.genesis_hash, None, cfgs[i].logger);
+               let network_graph = NetworkGraph::new(cfgs[i].chain_source.genesis_hash);
+               let net_graph_msg_handler = NetGraphMsgHandler::new(network_graph, None, cfgs[i].logger);
                nodes.push(Node{ chain_source: cfgs[i].chain_source,
                                 tx_broadcaster: cfgs[i].tx_broadcaster, chain_monitor: &cfgs[i].chain_monitor,
                                 keys_manager: &cfgs[i].keys_manager, node: &chan_mgrs[i], net_graph_msg_handler,
@@ -1381,6 +1564,13 @@ pub fn create_network<'a, 'b: 'a, 'c: 'b>(node_count: usize, cfgs: &'b Vec<NodeC
                })
        }
 
+       for i in 0..node_count {
+               for j in (i+1)..node_count {
+                       nodes[i].node.peer_connected(&nodes[j].node.get_our_node_id(), &msgs::Init { features: cfgs[j].features.clone() });
+                       nodes[j].node.peer_connected(&nodes[i].node.get_our_node_id(), &msgs::Init { features: cfgs[i].features.clone() });
+               }
+       }
+
        nodes
 }
 
@@ -1634,7 +1824,7 @@ macro_rules! handle_chan_reestablish_msgs {
 
 /// pending_htlc_adds includes both the holding cell and in-flight update_add_htlcs, whereas
 /// for claims/fails they are separated out.
-pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a, 'b, 'c>, send_funding_locked: (bool, bool), pending_htlc_adds: (i64, i64), pending_htlc_claims: (usize, usize), pending_cell_htlc_claims: (usize, usize), pending_cell_htlc_fails: (usize, usize), pending_raa: (bool, bool))  {
+pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a, 'b, 'c>, send_funding_locked: (bool, bool), pending_htlc_adds: (i64, i64), pending_htlc_claims: (usize, usize), pending_htlc_fails: (usize, usize), pending_cell_htlc_claims: (usize, usize), pending_cell_htlc_fails: (usize, usize), pending_raa: (bool, bool))  {
        node_a.node.peer_connected(&node_b.node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
        let reestablish_1 = get_chan_reestablish_msgs!(node_a, node_b);
        node_b.node.peer_connected(&node_a.node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
@@ -1688,8 +1878,10 @@ pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a,
        }
 
        // We don't yet support both needing updates, as that would require a different commitment dance:
-       assert!((pending_htlc_adds.0 == 0 && pending_htlc_claims.0 == 0 && pending_cell_htlc_claims.0 == 0 && pending_cell_htlc_fails.0 == 0) ||
-                       (pending_htlc_adds.1 == 0 && pending_htlc_claims.1 == 0 && pending_cell_htlc_claims.1 == 0 && pending_cell_htlc_fails.1 == 0));
+       assert!((pending_htlc_adds.0 == 0 && pending_htlc_claims.0 == 0 && pending_htlc_fails.0 == 0 &&
+                        pending_cell_htlc_claims.0 == 0 && pending_cell_htlc_fails.0 == 0) ||
+                       (pending_htlc_adds.1 == 0 && pending_htlc_claims.1 == 0 && pending_htlc_fails.1 == 0 &&
+                        pending_cell_htlc_claims.1 == 0 && pending_cell_htlc_fails.1 == 0));
 
        for chan_msgs in resp_1.drain(..) {
                if send_funding_locked.0 {
@@ -1712,7 +1904,7 @@ pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a,
                } else {
                        assert!(chan_msgs.1.is_none());
                }
-               if pending_htlc_adds.0 != 0 || pending_htlc_claims.0 != 0 || pending_cell_htlc_claims.0 != 0 || pending_cell_htlc_fails.0 != 0 {
+               if pending_htlc_adds.0 != 0 || pending_htlc_claims.0 != 0 || pending_htlc_fails.0 != 0 || pending_cell_htlc_claims.0 != 0 || pending_cell_htlc_fails.0 != 0 {
                        let commitment_update = chan_msgs.2.unwrap();
                        if pending_htlc_adds.0 != -1 { // We use -1 to denote a response commitment_signed
                                assert_eq!(commitment_update.update_add_htlcs.len(), pending_htlc_adds.0 as usize);
@@ -1720,7 +1912,7 @@ pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a,
                                assert!(commitment_update.update_add_htlcs.is_empty());
                        }
                        assert_eq!(commitment_update.update_fulfill_htlcs.len(), pending_htlc_claims.0 + pending_cell_htlc_claims.0);
-                       assert_eq!(commitment_update.update_fail_htlcs.len(), pending_cell_htlc_fails.0);
+                       assert_eq!(commitment_update.update_fail_htlcs.len(), pending_htlc_fails.0 + pending_cell_htlc_fails.0);
                        assert!(commitment_update.update_fail_malformed_htlcs.is_empty());
                        for update_add in commitment_update.update_add_htlcs {
                                node_a.node.handle_update_add_htlc(&node_b.node.get_our_node_id(), &update_add);
@@ -1769,13 +1961,13 @@ pub fn reconnect_nodes<'a, 'b, 'c>(node_a: &Node<'a, 'b, 'c>, node_b: &Node<'a,
                } else {
                        assert!(chan_msgs.1.is_none());
                }
-               if pending_htlc_adds.1 != 0 || pending_htlc_claims.1 != 0 || pending_cell_htlc_claims.1 != 0 || pending_cell_htlc_fails.1 != 0 {
+               if pending_htlc_adds.1 != 0 || pending_htlc_claims.1 != 0 || pending_htlc_fails.1 != 0 || pending_cell_htlc_claims.1 != 0 || pending_cell_htlc_fails.1 != 0 {
                        let commitment_update = chan_msgs.2.unwrap();
                        if pending_htlc_adds.1 != -1 { // We use -1 to denote a response commitment_signed
                                assert_eq!(commitment_update.update_add_htlcs.len(), pending_htlc_adds.1 as usize);
                        }
-                       assert_eq!(commitment_update.update_fulfill_htlcs.len(), pending_htlc_claims.0 + pending_cell_htlc_claims.0);
-                       assert_eq!(commitment_update.update_fail_htlcs.len(), pending_cell_htlc_fails.0);
+                       assert_eq!(commitment_update.update_fulfill_htlcs.len(), pending_htlc_claims.1 + pending_cell_htlc_claims.1);
+                       assert_eq!(commitment_update.update_fail_htlcs.len(), pending_htlc_fails.1 + pending_cell_htlc_fails.1);
                        assert!(commitment_update.update_fail_malformed_htlcs.is_empty());
                        for update_add in commitment_update.update_add_htlcs {
                                node_b.node.handle_update_add_htlc(&node_a.node.get_our_node_id(), &update_add);