_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
  11 //! monitor updates.
  12 //! There are a bunch of these as their handling is relatively error-prone so they are split out
  13 //! here. See also the chanmon_fail_consistency fuzz test.
  14
  15 use bitcoin::blockdata::block::{Block, BlockHeader};
  16 use bitcoin::hash_types::BlockHash;
  17 use bitcoin::network::constants::Network;
  18 use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdateErr};
  19 use chain::transaction::OutPoint;
  20 use chain::Listen;
  21 use chain::Watch;
  22 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSendFailure};
  23 use ln::features::{InitFeatures, InvoiceFeatures};
  24 use ln::msgs;
  25 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  26 use routing::router::get_route;
  27 use util::enforcing_trait_impls::EnforcingSigner;
  28 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  29 use util::errors::APIError;
  30 use util::ser::{ReadableArgs, Writeable};
  31
  32 use bitcoin::hashes::sha256::Hash as Sha256;
  33 use bitcoin::hashes::Hash;
  34
  35 use ln::functional_test_utils::*;
  36
  37 use util::test_utils;
  38
  39 // If persister_fail is true, we have the persister return a PermanentFailure
  40 // instead of the higher-level ChainMonitor.
  41 fn do_test_simple_monitor_permanent_update_fail(persister_fail: bool) {
  42         // Test that we handle a simple permanent monitor update failure
  43         let mut chanmon_cfgs = create_chanmon_cfgs(2);
  44         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  45         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  46         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  47         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
  48         let logger = test_utils::TestLogger::new();
  49
  50         let (_, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(&nodes[1]);
  51
  52         match persister_fail {
  53                 true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::PermanentFailure)),
  54                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::PermanentFailure))
  55         }
  56         let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
  57         let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
  58         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), true, APIError::ChannelUnavailable {..}, {});
  59         check_added_monitors!(nodes[0], 2);
  60
  61         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  62         assert_eq!(events_1.len(), 2);
  63         match events_1[0] {
  64                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  65                 _ => panic!("Unexpected event"),
  66         };
  67         match events_1[1] {
  68                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  69                 _ => panic!("Unexpected event"),
  70         };
  71
  72         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  73         // PaymentFailed event
  74
  75         assert_eq!(nodes[0].node.list_channels().len(), 0);
  76 }
  77
  78 #[test]
  79 fn test_monitor_and_persister_update_fail() {
  80         // Test that if both updating the `ChannelMonitor` and persisting the updated
  81         // `ChannelMonitor` fail, then the failure from updating the `ChannelMonitor`
  82         // one that gets returned.
  83         let chanmon_cfgs = create_chanmon_cfgs(2);
  84         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  85         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  86         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  87
  88         // Create some initial channel
  89         let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
  90         let outpoint = OutPoint { txid: chan.3.txid(), index: 0 };
  91
  92         // Rebalance the network to generate htlc in the two directions
  93         send_payment(&nodes[0], &vec!(&nodes[1])[..], 10_000_000, 10_000_000);
  94
  95         // Route an HTLC from node 0 to node 1 (but don't settle)
  96         let preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 9_000_000).0;
  97
  98         // Make a copy of the ChainMonitor so we can capture the error it returns on a
  99         // bogus update. Note that if instead we updated the nodes[0]'s ChainMonitor
 100         // directly, the node would fail to be `Drop`'d at the end because its
 101         // ChannelManager and ChainMonitor would be out of sync.
 102         let chain_source = test_utils::TestChainSource::new(Network::Testnet);
 103         let logger = test_utils::TestLogger::with_id(format!("node {}", 0));
 104         let persister = test_utils::TestPersister::new();
 105         let chain_mon = {
 106                 let monitors = nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap();
 107                 let monitor = monitors.get(&outpoint).unwrap();
 108                 let mut w = test_utils::TestVecWriter(Vec::new());
 109                 monitor.write(&mut w).unwrap();
 110                 let new_monitor = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
 111                         &mut ::std::io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
 112                 assert!(new_monitor == *monitor);
 113                 let chain_mon = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
 114                 assert!(chain_mon.watch_channel(outpoint, new_monitor).is_ok());
 115                 chain_mon
 116         };
 117         let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
 118         chain_mon.chain_monitor.block_connected(&Block { header, txdata: vec![] }, 200);
 119
 120         // Set the persister's return value to be a TemporaryFailure.
 121         persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 122
 123         // Try to update ChannelMonitor
 124         assert!(nodes[1].node.claim_funds(preimage, &None, 9_000_000));
 125         check_added_monitors!(nodes[1], 1);
 126         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 127         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 128         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 129         if let Some(ref mut channel) = nodes[0].node.channel_state.lock().unwrap().by_id.get_mut(&chan.2) {
 130                 if let Ok((_, _, _, update)) = channel.commitment_signed(&updates.commitment_signed, &node_cfgs[0].fee_estimator, &node_cfgs[0].logger) {
 131                         // Check that even though the persister is returning a TemporaryFailure,
 132                         // because the update is bogus, ultimately the error that's returned
 133                         // should be a PermanentFailure.
 134                         if let Err(ChannelMonitorUpdateErr::PermanentFailure) = chain_mon.chain_monitor.update_channel(outpoint, update.clone()) {} else { panic!("Expected monitor error to be permanent"); }
 135                         logger.assert_log_contains("lightning::chain::chainmonitor".to_string(), "Failed to persist channel monitor update: TemporaryFailure".to_string(), 1);
 136                         if let Ok(_) = nodes[0].chain_monitor.update_channel(outpoint, update) {} else { assert!(false); }
 137                 } else { assert!(false); }
 138         } else { assert!(false); };
 139
 140         check_added_monitors!(nodes[0], 1);
 141         let events = nodes[0].node.get_and_clear_pending_events();
 142         assert_eq!(events.len(), 1);
 143 }
 144
 145 #[test]
 146 fn test_simple_monitor_permanent_update_fail() {
 147         do_test_simple_monitor_permanent_update_fail(false);
 148
 149         // Test behavior when the persister returns a PermanentFailure.
 150         do_test_simple_monitor_permanent_update_fail(true);
 151 }
 152
 153 // If persister_fail is true, we have the persister return a TemporaryFailure instead of the
 154 // higher-level ChainMonitor.
 155 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail: bool) {
 156         // Test that we can recover from a simple temporary monitor update failure optionally with
 157         // a disconnect in between
 158         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 159         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 160         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 161         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 162         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 163         let logger = test_utils::TestLogger::new();
 164
 165         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(&nodes[1]);
 166
 167         match persister_fail {
 168                 true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure)),
 169                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure))
 170         }
 171
 172         {
 173                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 174                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 175                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), false, APIError::MonitorUpdateFailed, {});
 176                 check_added_monitors!(nodes[0], 1);
 177         }
 178
 179         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 180         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 181         assert_eq!(nodes[0].node.list_channels().len(), 1);
 182
 183         if disconnect {
 184                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 185                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 186                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 187         }
 188
 189         match persister_fail {
 190                 true => chanmon_cfgs[0].persister.set_update_ret(Ok(())),
 191                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()))
 192         }
 193         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 194         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 195         check_added_monitors!(nodes[0], 0);
 196
 197         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
 198         assert_eq!(events_2.len(), 1);
 199         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
 200         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 201         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 202         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 203
 204         expect_pending_htlcs_forwardable!(nodes[1]);
 205
 206         let events_3 = nodes[1].node.get_and_clear_pending_events();
 207         assert_eq!(events_3.len(), 1);
 208         match events_3[0] {
 209                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
 210                         assert_eq!(payment_hash_1, *payment_hash);
 211                         assert_eq!(*payment_secret, None);
 212                         assert_eq!(amt, 1000000);
 213                 },
 214                 _ => panic!("Unexpected event"),
 215         }
 216
 217         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 218
 219         // Now set it to failed again...
 220         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(&nodes[1]);
 221         {
 222                 match persister_fail {
 223                         true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure)),
 224                         false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure))
 225                 }
 226                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 227                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 228                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
 229                 check_added_monitors!(nodes[0], 1);
 230         }
 231
 232         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 233         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 234         assert_eq!(nodes[0].node.list_channels().len(), 1);
 235
 236         if disconnect {
 237                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 238                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 239                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 240         }
 241
 242         // ...and make sure we can force-close a frozen channel
 243         nodes[0].node.force_close_channel(&channel_id).unwrap();
 244         check_added_monitors!(nodes[0], 1);
 245         check_closed_broadcast!(nodes[0], true);
 246
 247         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 248         // PaymentFailed event
 249
 250         assert_eq!(nodes[0].node.list_channels().len(), 0);
 251 }
 252
 253 #[test]
 254 fn test_simple_monitor_temporary_update_fail() {
 255         do_test_simple_monitor_temporary_update_fail(false, false);
 256         do_test_simple_monitor_temporary_update_fail(true, false);
 257
 258         // Test behavior when the persister returns a TemporaryFailure.
 259         do_test_simple_monitor_temporary_update_fail(false, true);
 260         do_test_simple_monitor_temporary_update_fail(true, true);
 261 }
 262
 263 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 264         let disconnect_flags = 8 | 16;
 265
 266         // Test that we can recover from a temporary monitor update failure with some in-flight
 267         // HTLCs going on at the same time potentially with some disconnection thrown in.
 268         // * First we route a payment, then get a temporary monitor update failure when trying to
 269         //   route a second payment. We then claim the first payment.
 270         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 271         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 272         //   the ChannelMonitor on a watchtower).
 273         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 274         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 275         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 276         //   disconnect_count & !disconnect_flags is 0).
 277         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 278         //   through message sending, potentially disconnect/reconnecting multiple times based on
 279         //   disconnect_count, to get the update_fulfill_htlc through.
 280         // * We then walk through more message exchanges to get the original update_add_htlc
 281         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 282         //   disconnect/reconnecting based on disconnect_count.
 283         let chanmon_cfgs = create_chanmon_cfgs(2);
 284         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 285         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 286         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 287         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 288         let logger = test_utils::TestLogger::new();
 289
 290         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 291
 292         // Now try to send a second payment which will fail to send
 293         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
 294         {
 295                 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 296                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 297                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 298                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
 299                 check_added_monitors!(nodes[0], 1);
 300         }
 301
 302         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 303         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 304         assert_eq!(nodes[0].node.list_channels().len(), 1);
 305
 306         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 307         // but nodes[0] won't respond since it is frozen.
 308         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
 309         check_added_monitors!(nodes[1], 1);
 310         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 311         assert_eq!(events_2.len(), 1);
 312         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 313                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 314                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 315                         assert!(update_add_htlcs.is_empty());
 316                         assert_eq!(update_fulfill_htlcs.len(), 1);
 317                         assert!(update_fail_htlcs.is_empty());
 318                         assert!(update_fail_malformed_htlcs.is_empty());
 319                         assert!(update_fee.is_none());
 320
 321                         if (disconnect_count & 16) == 0 {
 322                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 323                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 324                                 assert_eq!(events_3.len(), 1);
 325                                 match events_3[0] {
 326                                         Event::PaymentSent { ref payment_preimage } => {
 327                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 328                                         },
 329                                         _ => panic!("Unexpected event"),
 330                                 }
 331
 332                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 333                                 check_added_monitors!(nodes[0], 1);
 334                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 335                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 336                         }
 337
 338                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 339                 },
 340                 _ => panic!("Unexpected event"),
 341         };
 342
 343         if disconnect_count & !disconnect_flags > 0 {
 344                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 345                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 346         }
 347
 348         // Now fix monitor updating...
 349         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 350         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 351         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 352         check_added_monitors!(nodes[0], 0);
 353
 354         macro_rules! disconnect_reconnect_peers { () => { {
 355                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 356                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 357
 358                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 359                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 360                 assert_eq!(reestablish_1.len(), 1);
 361                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 362                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 363                 assert_eq!(reestablish_2.len(), 1);
 364
 365                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 366                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 367                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 368                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 369
 370                 assert!(as_resp.0.is_none());
 371                 assert!(bs_resp.0.is_none());
 372
 373                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 374         } } }
 375
 376         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 377                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 378                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 379
 380                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 381                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 382                 assert_eq!(reestablish_1.len(), 1);
 383                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 384                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 385                 assert_eq!(reestablish_2.len(), 1);
 386
 387                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 388                 check_added_monitors!(nodes[0], 0);
 389                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 390                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 391                 check_added_monitors!(nodes[1], 0);
 392                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 393
 394                 assert!(as_resp.0.is_none());
 395                 assert!(bs_resp.0.is_none());
 396
 397                 assert!(bs_resp.1.is_none());
 398                 if (disconnect_count & 16) == 0 {
 399                         assert!(bs_resp.2.is_none());
 400
 401                         assert!(as_resp.1.is_some());
 402                         assert!(as_resp.2.is_some());
 403                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 404                 } else {
 405                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 406                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 407                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 408                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 409                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 410                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 411
 412                         assert!(as_resp.1.is_none());
 413
 414                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 415                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 416                         assert_eq!(events_3.len(), 1);
 417                         match events_3[0] {
 418                                 Event::PaymentSent { ref payment_preimage } => {
 419                                         assert_eq!(*payment_preimage, payment_preimage_1);
 420                                 },
 421                                 _ => panic!("Unexpected event"),
 422                         }
 423
 424                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 425                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 426                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 427                         check_added_monitors!(nodes[0], 1);
 428
 429                         as_resp.1 = Some(as_resp_raa);
 430                         bs_resp.2 = None;
 431                 }
 432
 433                 if disconnect_count & !disconnect_flags > 1 {
 434                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 435
 436                         if (disconnect_count & 16) == 0 {
 437                                 assert!(reestablish_1 == second_reestablish_1);
 438                                 assert!(reestablish_2 == second_reestablish_2);
 439                         }
 440                         assert!(as_resp == second_as_resp);
 441                         assert!(bs_resp == second_bs_resp);
 442                 }
 443
 444                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 445         } else {
 446                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 447                 assert_eq!(events_4.len(), 2);
 448                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 449                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 450                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 451                                 msg.clone()
 452                         },
 453                         _ => panic!("Unexpected event"),
 454                 })
 455         };
 456
 457         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 458
 459         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 460         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 461         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 462         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 463         check_added_monitors!(nodes[1], 1);
 464
 465         if disconnect_count & !disconnect_flags > 2 {
 466                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 467
 468                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 469                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 470
 471                 assert!(as_resp.2.is_none());
 472                 assert!(bs_resp.2.is_none());
 473         }
 474
 475         let as_commitment_update;
 476         let bs_second_commitment_update;
 477
 478         macro_rules! handle_bs_raa { () => {
 479                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 480                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 481                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 482                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 483                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 484                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 485                 assert!(as_commitment_update.update_fee.is_none());
 486                 check_added_monitors!(nodes[0], 1);
 487         } }
 488
 489         macro_rules! handle_initial_raa { () => {
 490                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 491                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 492                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 493                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 494                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 495                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 496                 assert!(bs_second_commitment_update.update_fee.is_none());
 497                 check_added_monitors!(nodes[1], 1);
 498         } }
 499
 500         if (disconnect_count & 8) == 0 {
 501                 handle_bs_raa!();
 502
 503                 if disconnect_count & !disconnect_flags > 3 {
 504                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 505
 506                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 507                         assert!(bs_resp.1.is_none());
 508
 509                         assert!(as_resp.2.unwrap() == as_commitment_update);
 510                         assert!(bs_resp.2.is_none());
 511
 512                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 513                 }
 514
 515                 handle_initial_raa!();
 516
 517                 if disconnect_count & !disconnect_flags > 4 {
 518                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 519
 520                         assert!(as_resp.1.is_none());
 521                         assert!(bs_resp.1.is_none());
 522
 523                         assert!(as_resp.2.unwrap() == as_commitment_update);
 524                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 525                 }
 526         } else {
 527                 handle_initial_raa!();
 528
 529                 if disconnect_count & !disconnect_flags > 3 {
 530                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 531
 532                         assert!(as_resp.1.is_none());
 533                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 534
 535                         assert!(as_resp.2.is_none());
 536                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 537
 538                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 539                 }
 540
 541                 handle_bs_raa!();
 542
 543                 if disconnect_count & !disconnect_flags > 4 {
 544                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 545
 546                         assert!(as_resp.1.is_none());
 547                         assert!(bs_resp.1.is_none());
 548
 549                         assert!(as_resp.2.unwrap() == as_commitment_update);
 550                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 551                 }
 552         }
 553
 554         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 555         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 556         // No commitment_signed so get_event_msg's assert(len == 1) passes
 557         check_added_monitors!(nodes[0], 1);
 558
 559         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 560         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 561         // No commitment_signed so get_event_msg's assert(len == 1) passes
 562         check_added_monitors!(nodes[1], 1);
 563
 564         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 565         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 566         check_added_monitors!(nodes[1], 1);
 567
 568         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 569         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 570         check_added_monitors!(nodes[0], 1);
 571
 572         expect_pending_htlcs_forwardable!(nodes[1]);
 573
 574         let events_5 = nodes[1].node.get_and_clear_pending_events();
 575         assert_eq!(events_5.len(), 1);
 576         match events_5[0] {
 577                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
 578                         assert_eq!(payment_hash_2, *payment_hash);
 579                         assert_eq!(*payment_secret, None);
 580                         assert_eq!(amt, 1000000);
 581                 },
 582                 _ => panic!("Unexpected event"),
 583         }
 584
 585         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 586 }
 587
 588 #[test]
 589 fn test_monitor_temporary_update_fail_a() {
 590         do_test_monitor_temporary_update_fail(0);
 591         do_test_monitor_temporary_update_fail(1);
 592         do_test_monitor_temporary_update_fail(2);
 593         do_test_monitor_temporary_update_fail(3);
 594         do_test_monitor_temporary_update_fail(4);
 595         do_test_monitor_temporary_update_fail(5);
 596 }
 597
 598 #[test]
 599 fn test_monitor_temporary_update_fail_b() {
 600         do_test_monitor_temporary_update_fail(2 | 8);
 601         do_test_monitor_temporary_update_fail(3 | 8);
 602         do_test_monitor_temporary_update_fail(4 | 8);
 603         do_test_monitor_temporary_update_fail(5 | 8);
 604 }
 605
 606 #[test]
 607 fn test_monitor_temporary_update_fail_c() {
 608         do_test_monitor_temporary_update_fail(1 | 16);
 609         do_test_monitor_temporary_update_fail(2 | 16);
 610         do_test_monitor_temporary_update_fail(3 | 16);
 611         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 612         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 613 }
 614
 615 #[test]
 616 fn test_monitor_update_fail_cs() {
 617         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 618         let chanmon_cfgs = create_chanmon_cfgs(2);
 619         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 620         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 621         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 622         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 623         let logger = test_utils::TestLogger::new();
 624
 625         let (payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
 626         {
 627                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 628                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 629                 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
 630                 check_added_monitors!(nodes[0], 1);
 631         }
 632
 633         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 634         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 635
 636         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 637         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 638         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 639         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 640         check_added_monitors!(nodes[1], 1);
 641         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 642
 643         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 644         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 645         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 646         check_added_monitors!(nodes[1], 0);
 647         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 648         assert_eq!(responses.len(), 2);
 649
 650         match responses[0] {
 651                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 652                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 653                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 654                         check_added_monitors!(nodes[0], 1);
 655                 },
 656                 _ => panic!("Unexpected event"),
 657         }
 658         match responses[1] {
 659                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 660                         assert!(updates.update_add_htlcs.is_empty());
 661                         assert!(updates.update_fulfill_htlcs.is_empty());
 662                         assert!(updates.update_fail_htlcs.is_empty());
 663                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 664                         assert!(updates.update_fee.is_none());
 665                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 666
 667                         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 668                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 669                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 670                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 671                         check_added_monitors!(nodes[0], 1);
 672                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 673                 },
 674                 _ => panic!("Unexpected event"),
 675         }
 676
 677         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 678         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 679         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 680         check_added_monitors!(nodes[0], 0);
 681
 682         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 683         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 684         check_added_monitors!(nodes[1], 1);
 685
 686         expect_pending_htlcs_forwardable!(nodes[1]);
 687
 688         let events = nodes[1].node.get_and_clear_pending_events();
 689         assert_eq!(events.len(), 1);
 690         match events[0] {
 691                 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
 692                         assert_eq!(payment_hash, our_payment_hash);
 693                         assert_eq!(payment_secret, None);
 694                         assert_eq!(amt, 1000000);
 695                 },
 696                 _ => panic!("Unexpected event"),
 697         };
 698
 699         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 700 }
 701
 702 #[test]
 703 fn test_monitor_update_fail_no_rebroadcast() {
 704         // Tests handling of a monitor update failure when no message rebroadcasting on
 705         // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
 706         // fuzz tests.
 707         let chanmon_cfgs = create_chanmon_cfgs(2);
 708         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 709         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 710         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 711         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 712         let logger = test_utils::TestLogger::new();
 713
 714         let (payment_preimage_1, our_payment_hash, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
 715         {
 716                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 717                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 718                 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
 719                 check_added_monitors!(nodes[0], 1);
 720         }
 721
 722         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 723         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 724         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 725
 726         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 727         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 728         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 729         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 730         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 731         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 732         check_added_monitors!(nodes[1], 1);
 733
 734         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 735         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 736         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 737         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 738         check_added_monitors!(nodes[1], 0);
 739         expect_pending_htlcs_forwardable!(nodes[1]);
 740
 741         let events = nodes[1].node.get_and_clear_pending_events();
 742         assert_eq!(events.len(), 1);
 743         match events[0] {
 744                 Event::PaymentReceived { payment_hash, .. } => {
 745                         assert_eq!(payment_hash, our_payment_hash);
 746                 },
 747                 _ => panic!("Unexpected event"),
 748         }
 749
 750         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 751 }
 752
 753 #[test]
 754 fn test_monitor_update_raa_while_paused() {
 755         // Tests handling of an RAA while monitor updating has already been marked failed.
 756         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 757         let chanmon_cfgs = create_chanmon_cfgs(2);
 758         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 759         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 760         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 761         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 762         let logger = test_utils::TestLogger::new();
 763
 764         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 765         let (payment_preimage_1, our_payment_hash_1, our_payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
 766         {
 767                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 768                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 769                 nodes[0].node.send_payment(&route, our_payment_hash_1, &None).unwrap();
 770                 check_added_monitors!(nodes[0], 1);
 771         }
 772         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 773
 774         let (payment_preimage_2, our_payment_hash_2, our_payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
 775         {
 776                 let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
 777                 let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 778                 nodes[1].node.send_payment(&route, our_payment_hash_2, &None).unwrap();
 779                 check_added_monitors!(nodes[1], 1);
 780         }
 781         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 782
 783         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 784         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 785         check_added_monitors!(nodes[1], 1);
 786         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 787
 788         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 789         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 790         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 791         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 792         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 793         check_added_monitors!(nodes[0], 1);
 794
 795         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 796         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 797         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 798         check_added_monitors!(nodes[0], 1);
 799
 800         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 801         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 802         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 803         check_added_monitors!(nodes[0], 0);
 804
 805         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 806         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 807         check_added_monitors!(nodes[1], 1);
 808         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 809
 810         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 811         check_added_monitors!(nodes[1], 1);
 812         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 813
 814         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 815         check_added_monitors!(nodes[0], 1);
 816         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 817
 818         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 819         check_added_monitors!(nodes[0], 1);
 820         expect_pending_htlcs_forwardable!(nodes[0]);
 821         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 822
 823         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 824         check_added_monitors!(nodes[1], 1);
 825         expect_pending_htlcs_forwardable!(nodes[1]);
 826         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 827
 828         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 829         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 830 }
 831
 832 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 833         // Tests handling of a monitor update failure when processing an incoming RAA
 834         let chanmon_cfgs = create_chanmon_cfgs(3);
 835         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 836         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 837         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 838         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
 839         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
 840         let logger = test_utils::TestLogger::new();
 841
 842         // Rebalance a bit so that we can send backwards from 2 to 1.
 843         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 844
 845         // Route a first payment that we'll fail backwards
 846         let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 847
 848         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 849         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
 850         expect_pending_htlcs_forwardable!(nodes[2]);
 851         check_added_monitors!(nodes[2], 1);
 852
 853         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 854         assert!(updates.update_add_htlcs.is_empty());
 855         assert!(updates.update_fulfill_htlcs.is_empty());
 856         assert_eq!(updates.update_fail_htlcs.len(), 1);
 857         assert!(updates.update_fail_malformed_htlcs.is_empty());
 858         assert!(updates.update_fee.is_none());
 859         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 860
 861         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 862         check_added_monitors!(nodes[0], 0);
 863
 864         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 865         // holding cell.
 866         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[2]);
 867         {
 868                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 869                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 870                 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
 871                 check_added_monitors!(nodes[0], 1);
 872         }
 873
 874         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 875         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 876         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 877
 878         expect_pending_htlcs_forwardable!(nodes[1]);
 879         check_added_monitors!(nodes[1], 0);
 880         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 881
 882         // Now fail monitor updating.
 883         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 884         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 885         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 886         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 887         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 888         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 889         check_added_monitors!(nodes[1], 1);
 890
 891         // Attempt to forward a third payment but fail due to the second channel being unavailable
 892         // for forwarding.
 893         let (_, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[2]);
 894         {
 895                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 896                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 897                 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
 898                 check_added_monitors!(nodes[0], 1);
 899         }
 900
 901         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(())); // We succeed in updating the monitor for the first channel
 902         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 903         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 904         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 905         check_added_monitors!(nodes[1], 0);
 906
 907         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 908         assert_eq!(events_2.len(), 1);
 909         match events_2.remove(0) {
 910                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 911                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 912                         assert!(updates.update_fulfill_htlcs.is_empty());
 913                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 914                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 915                         assert!(updates.update_add_htlcs.is_empty());
 916                         assert!(updates.update_fee.is_none());
 917
 918                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 919                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 920
 921                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 922                         assert_eq!(msg_events.len(), 1);
 923                         match msg_events[0] {
 924                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 925                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 926                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 927                                 },
 928                                 _ => panic!("Unexpected event"),
 929                         }
 930
 931                         let events = nodes[0].node.get_and_clear_pending_events();
 932                         assert_eq!(events.len(), 1);
 933                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 934                                 assert_eq!(payment_hash, payment_hash_3);
 935                                 assert!(!rejected_by_dest);
 936                         } else { panic!("Unexpected event!"); }
 937                 },
 938                 _ => panic!("Unexpected event type!"),
 939         };
 940
 941         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 942                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 943                 let (payment_preimage_4, payment_hash_4, payment_secret_4) = get_payment_preimage_hash!(nodes[0]);
 944                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
 945                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 946                 nodes[2].node.send_payment(&route, payment_hash_4, &None).unwrap();
 947                 check_added_monitors!(nodes[2], 1);
 948
 949                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 950                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 951                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 952                 check_added_monitors!(nodes[1], 1);
 953                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 954                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 955                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 956                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 957                 (Some(payment_preimage_4), Some(payment_hash_4))
 958         } else { (None, None) };
 959
 960         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 961         // update_add update.
 962         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 963         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
 964         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 965         check_added_monitors!(nodes[1], 0);
 966         expect_pending_htlcs_forwardable!(nodes[1]);
 967         check_added_monitors!(nodes[1], 1);
 968
 969         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 970         if test_ignore_second_cs {
 971                 assert_eq!(events_3.len(), 3);
 972         } else {
 973                 assert_eq!(events_3.len(), 2);
 974         }
 975
 976         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 977         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 978         let messages_a = match events_3.pop().unwrap() {
 979                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 980                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 981                         assert!(updates.update_fulfill_htlcs.is_empty());
 982                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 983                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 984                         assert!(updates.update_add_htlcs.is_empty());
 985                         assert!(updates.update_fee.is_none());
 986                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 987                 },
 988                 _ => panic!("Unexpected event type!"),
 989         };
 990         let raa = if test_ignore_second_cs {
 991                 match events_3.remove(1) {
 992                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 993                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 994                                 Some(msg.clone())
 995                         },
 996                         _ => panic!("Unexpected event"),
 997                 }
 998         } else { None };
 999         let send_event_b = SendEvent::from_event(events_3.remove(0));
1000         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
1001
1002         // Now deliver the new messages...
1003
1004         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
1005         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
1006         let events_4 = nodes[0].node.get_and_clear_pending_events();
1007         assert_eq!(events_4.len(), 1);
1008         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
1009                 assert_eq!(payment_hash, payment_hash_1);
1010                 assert!(rejected_by_dest);
1011         } else { panic!("Unexpected event!"); }
1012
1013         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
1014         if test_ignore_second_cs {
1015                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
1016                 check_added_monitors!(nodes[2], 1);
1017                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1018                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
1019                 check_added_monitors!(nodes[2], 1);
1020                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1021                 assert!(bs_cs.update_add_htlcs.is_empty());
1022                 assert!(bs_cs.update_fail_htlcs.is_empty());
1023                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
1024                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
1025                 assert!(bs_cs.update_fee.is_none());
1026
1027                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
1028                 check_added_monitors!(nodes[1], 1);
1029                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1030                 assert!(as_cs.update_add_htlcs.is_empty());
1031                 assert!(as_cs.update_fail_htlcs.is_empty());
1032                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
1033                 assert!(as_cs.update_fulfill_htlcs.is_empty());
1034                 assert!(as_cs.update_fee.is_none());
1035
1036                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
1037                 check_added_monitors!(nodes[1], 1);
1038                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
1039
1040                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
1041                 check_added_monitors!(nodes[2], 1);
1042                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1043
1044                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
1045                 check_added_monitors!(nodes[2], 1);
1046                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
1047
1048                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
1049                 check_added_monitors!(nodes[1], 1);
1050                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1051         } else {
1052                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
1053         }
1054
1055         expect_pending_htlcs_forwardable!(nodes[2]);
1056
1057         let events_6 = nodes[2].node.get_and_clear_pending_events();
1058         assert_eq!(events_6.len(), 1);
1059         match events_6[0] {
1060                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
1061                 _ => panic!("Unexpected event"),
1062         };
1063
1064         if test_ignore_second_cs {
1065                 expect_pending_htlcs_forwardable!(nodes[1]);
1066                 check_added_monitors!(nodes[1], 1);
1067
1068                 send_event = SendEvent::from_node(&nodes[1]);
1069                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
1070                 assert_eq!(send_event.msgs.len(), 1);
1071                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
1072                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
1073
1074                 expect_pending_htlcs_forwardable!(nodes[0]);
1075
1076                 let events_9 = nodes[0].node.get_and_clear_pending_events();
1077                 assert_eq!(events_9.len(), 1);
1078                 match events_9[0] {
1079                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
1080                         _ => panic!("Unexpected event"),
1081                 };
1082                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
1083         }
1084
1085         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
1086 }
1087
1088 #[test]
1089 fn test_monitor_update_fail_raa() {
1090         do_test_monitor_update_fail_raa(false);
1091         do_test_monitor_update_fail_raa(true);
1092 }
1093
1094 #[test]
1095 fn test_monitor_update_fail_reestablish() {
1096         // Simple test for message retransmission after monitor update failure on
1097         // channel_reestablish generating a monitor update (which comes from freeing holding cell
1098         // HTLCs).
1099         let chanmon_cfgs = create_chanmon_cfgs(3);
1100         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1101         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1102         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1103         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1104         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1105
1106         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1107
1108         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1109         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1110
1111         assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
1112         check_added_monitors!(nodes[2], 1);
1113         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1114         assert!(updates.update_add_htlcs.is_empty());
1115         assert!(updates.update_fail_htlcs.is_empty());
1116         assert!(updates.update_fail_malformed_htlcs.is_empty());
1117         assert!(updates.update_fee.is_none());
1118         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1119         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1120         check_added_monitors!(nodes[1], 1);
1121         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1122         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
1123
1124         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1125         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1126         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1127
1128         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1129         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1130
1131         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1132
1133         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1134         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1135         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1136         check_added_monitors!(nodes[1], 1);
1137
1138         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1139         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1140
1141         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1142         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1143
1144         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
1145         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
1146
1147         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1148
1149         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1150         check_added_monitors!(nodes[1], 0);
1151         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1152
1153         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1154         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1155         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1156         check_added_monitors!(nodes[1], 0);
1157
1158         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1159         assert!(updates.update_add_htlcs.is_empty());
1160         assert!(updates.update_fail_htlcs.is_empty());
1161         assert!(updates.update_fail_malformed_htlcs.is_empty());
1162         assert!(updates.update_fee.is_none());
1163         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1164         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1165         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1166
1167         let events = nodes[0].node.get_and_clear_pending_events();
1168         assert_eq!(events.len(), 1);
1169         match events[0] {
1170                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1171                 _ => panic!("Unexpected event"),
1172         }
1173 }
1174
1175 #[test]
1176 fn raa_no_response_awaiting_raa_state() {
1177         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1178         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1179         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1180         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1181         let chanmon_cfgs = create_chanmon_cfgs(2);
1182         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1183         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1184         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1185         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1186         let logger = test_utils::TestLogger::new();
1187
1188         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1189         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1190         let (payment_preimage_3, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[1]);
1191
1192         // Queue up two payments - one will be delivered right away, one immediately goes into the
1193         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1194         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1195         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1196         // generation during RAA while in monitor-update-failed state.
1197         {
1198                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1199                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1200                 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1201                 check_added_monitors!(nodes[0], 1);
1202                 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1203                 check_added_monitors!(nodes[0], 0);
1204         }
1205
1206         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1207         assert_eq!(events.len(), 1);
1208         let payment_event = SendEvent::from_event(events.pop().unwrap());
1209         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1210         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1211         check_added_monitors!(nodes[1], 1);
1212
1213         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1214         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1215         check_added_monitors!(nodes[0], 1);
1216         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1217         assert_eq!(events.len(), 1);
1218         let payment_event = SendEvent::from_event(events.pop().unwrap());
1219
1220         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1221         check_added_monitors!(nodes[0], 1);
1222         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1223
1224         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1225         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1226         // then restore channel monitor updates.
1227         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1228         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1229         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1230         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1231         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1232         check_added_monitors!(nodes[1], 1);
1233
1234         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1235         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1236         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1237         check_added_monitors!(nodes[1], 1);
1238
1239         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1240         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1241         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1242         // nodes[1] should be AwaitingRAA here!
1243         check_added_monitors!(nodes[1], 0);
1244         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1245         expect_pending_htlcs_forwardable!(nodes[1]);
1246         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1247
1248         // We send a third payment here, which is somewhat of a redundant test, but the
1249         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1250         // commitment transaction states) whereas here we can explicitly check for it.
1251         {
1252                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1253                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1254                 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
1255                 check_added_monitors!(nodes[0], 0);
1256                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1257         }
1258         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1259         check_added_monitors!(nodes[0], 1);
1260         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1261         assert_eq!(events.len(), 1);
1262         let payment_event = SendEvent::from_event(events.pop().unwrap());
1263
1264         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1265         check_added_monitors!(nodes[0], 1);
1266         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1267
1268         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1269         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1270         check_added_monitors!(nodes[1], 1);
1271         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1272
1273         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1274         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1275         check_added_monitors!(nodes[1], 1);
1276         expect_pending_htlcs_forwardable!(nodes[1]);
1277         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1278         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1279
1280         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1281         check_added_monitors!(nodes[0], 1);
1282
1283         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1284         check_added_monitors!(nodes[0], 1);
1285         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1286
1287         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1288         check_added_monitors!(nodes[1], 1);
1289         expect_pending_htlcs_forwardable!(nodes[1]);
1290         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1291
1292         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1293         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1294         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1295 }
1296
1297 #[test]
1298 fn claim_while_disconnected_monitor_update_fail() {
1299         // Test for claiming a payment while disconnected and then having the resulting
1300         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1301         // contrived case for nodes with network instability.
1302         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1303         // code introduced a regression in this test (specifically, this caught a removal of the
1304         // channel_reestablish handling ensuring the order was sensical given the messages used).
1305         let chanmon_cfgs = create_chanmon_cfgs(2);
1306         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1307         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1308         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1309         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1310         let logger = test_utils::TestLogger::new();
1311
1312         // Forward a payment for B to claim
1313         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1314
1315         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1316         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1317
1318         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1319         check_added_monitors!(nodes[1], 1);
1320
1321         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1322         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1323
1324         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1325         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1326
1327         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1328         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1329
1330         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1331         // update.
1332         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1333
1334         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1335         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1336         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1337         check_added_monitors!(nodes[1], 1);
1338         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1339
1340         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1341         // the monitor still failed
1342         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1343         {
1344                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1345                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1346                 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1347                 check_added_monitors!(nodes[0], 1);
1348         }
1349
1350         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1351         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1352         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1353         check_added_monitors!(nodes[1], 1);
1354         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1355         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1356         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1357         // until we've channel_monitor_update'd and updated for the new commitment transaction.
1358
1359         // Now un-fail the monitor, which will result in B sending its original commitment update,
1360         // receiving the commitment update from A, and the resulting commitment dances.
1361         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1362         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1363         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1364         check_added_monitors!(nodes[1], 0);
1365
1366         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1367         assert_eq!(bs_msgs.len(), 2);
1368
1369         match bs_msgs[0] {
1370                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1371                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1372                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1373                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1374                         check_added_monitors!(nodes[0], 1);
1375
1376                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1377                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1378                         check_added_monitors!(nodes[1], 1);
1379                 },
1380                 _ => panic!("Unexpected event"),
1381         }
1382
1383         match bs_msgs[1] {
1384                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1385                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1386                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1387                         check_added_monitors!(nodes[0], 1);
1388                 },
1389                 _ => panic!("Unexpected event"),
1390         }
1391
1392         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1393
1394         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1395         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1396         check_added_monitors!(nodes[0], 1);
1397         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1398
1399         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1400         check_added_monitors!(nodes[1], 1);
1401         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1402         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1403         check_added_monitors!(nodes[1], 1);
1404
1405         expect_pending_htlcs_forwardable!(nodes[1]);
1406         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1407
1408         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1409         check_added_monitors!(nodes[0], 1);
1410
1411         let events = nodes[0].node.get_and_clear_pending_events();
1412         assert_eq!(events.len(), 1);
1413         match events[0] {
1414                 Event::PaymentSent { ref payment_preimage } => {
1415                         assert_eq!(*payment_preimage, payment_preimage_1);
1416                 },
1417                 _ => panic!("Unexpected event"),
1418         }
1419
1420         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1421 }
1422
1423 #[test]
1424 fn monitor_failed_no_reestablish_response() {
1425         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1426         // response to a commitment_signed.
1427         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1428         // debug_assert!() failure in channel_reestablish handling.
1429         let chanmon_cfgs = create_chanmon_cfgs(2);
1430         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1431         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1432         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1433         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1434         let logger = test_utils::TestLogger::new();
1435
1436         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1437         // on receipt).
1438         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1439         {
1440                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1441                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1442                 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1443                 check_added_monitors!(nodes[0], 1);
1444         }
1445
1446         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1447         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1448         assert_eq!(events.len(), 1);
1449         let payment_event = SendEvent::from_event(events.pop().unwrap());
1450         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1451         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1452         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1453         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1454         check_added_monitors!(nodes[1], 1);
1455
1456         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1457         // is still failing to update monitors.
1458         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1459         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1460
1461         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1462         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1463
1464         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1465         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1466
1467         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1468         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1469
1470         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1471         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1472         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1473         check_added_monitors!(nodes[1], 0);
1474         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1475
1476         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1477         check_added_monitors!(nodes[0], 1);
1478         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1479         check_added_monitors!(nodes[0], 1);
1480
1481         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1482         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1483         check_added_monitors!(nodes[1], 1);
1484
1485         expect_pending_htlcs_forwardable!(nodes[1]);
1486         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1487
1488         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1489 }
1490
1491 #[test]
1492 fn first_message_on_recv_ordering() {
1493         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1494         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1495         // a commitment_signed which needs to send an RAA first.
1496         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1497         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1498         // response. To do this, we start routing two payments, with the final RAA for the first being
1499         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1500         // have no pending response but will want to send a RAA/CS (with the updates for the second
1501         // payment applied).
1502         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1503         let chanmon_cfgs = create_chanmon_cfgs(2);
1504         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1505         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1506         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1507         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1508         let logger = test_utils::TestLogger::new();
1509
1510         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1511         // can deliver it and fail the monitor update.
1512         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1513         {
1514                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1515                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1516                 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1517                 check_added_monitors!(nodes[0], 1);
1518         }
1519
1520         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1521         assert_eq!(events.len(), 1);
1522         let payment_event = SendEvent::from_event(events.pop().unwrap());
1523         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1524         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1525         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1526         check_added_monitors!(nodes[1], 1);
1527         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1528
1529         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1530         check_added_monitors!(nodes[0], 1);
1531         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1532         check_added_monitors!(nodes[0], 1);
1533
1534         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1535
1536         // Route the second payment, generating an update_add_htlc/commitment_signed
1537         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1538         {
1539                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1540                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1541                 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1542                 check_added_monitors!(nodes[0], 1);
1543         }
1544         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1545         assert_eq!(events.len(), 1);
1546         let payment_event = SendEvent::from_event(events.pop().unwrap());
1547         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1548
1549         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1550
1551         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1552         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1553         // to the next message also tests resetting the delivery order.
1554         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1555         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1556         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1557         check_added_monitors!(nodes[1], 1);
1558
1559         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1560         // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1561         // appropriate HTLC acceptance).
1562         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1563         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1564         check_added_monitors!(nodes[1], 1);
1565         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1566         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1567
1568         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1569         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1570         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1571         check_added_monitors!(nodes[1], 0);
1572
1573         expect_pending_htlcs_forwardable!(nodes[1]);
1574         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1575
1576         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1577         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1578         check_added_monitors!(nodes[0], 1);
1579         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1580         check_added_monitors!(nodes[0], 1);
1581
1582         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1583         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1584         check_added_monitors!(nodes[1], 1);
1585
1586         expect_pending_htlcs_forwardable!(nodes[1]);
1587         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1588
1589         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1590         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1591 }
1592
1593 #[test]
1594 fn test_monitor_update_fail_claim() {
1595         // Basic test for monitor update failures when processing claim_funds calls.
1596         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1597         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1598         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1599         // updating and claim the payment on B.
1600         let chanmon_cfgs = create_chanmon_cfgs(3);
1601         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1602         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1603         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1604         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1605         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1606         let logger = test_utils::TestLogger::new();
1607
1608         // Rebalance a bit so that we can send backwards from 3 to 2.
1609         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1610
1611         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1612
1613         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1614         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1615         check_added_monitors!(nodes[1], 1);
1616
1617         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
1618         {
1619                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1620                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1621                 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1622                 check_added_monitors!(nodes[2], 1);
1623         }
1624
1625         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1626         // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1627         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1628
1629         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1630         assert_eq!(events.len(), 1);
1631         let payment_event = SendEvent::from_event(events.pop().unwrap());
1632         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1633         let events = nodes[1].node.get_and_clear_pending_msg_events();
1634         assert_eq!(events.len(), 0);
1635         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1636         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1637
1638         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1639         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1640         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1641
1642         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1643         assert_eq!(msg_events.len(), 1);
1644         match msg_events[0] {
1645                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1646                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1647                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1648                 },
1649                 _ => panic!("Unexpected event"),
1650         }
1651
1652         let events = nodes[2].node.get_and_clear_pending_events();
1653         assert_eq!(events.len(), 1);
1654         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1655                 assert_eq!(payment_hash, payment_hash_2);
1656                 assert!(!rejected_by_dest);
1657         } else { panic!("Unexpected event!"); }
1658
1659         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1660         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1661         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1662         check_added_monitors!(nodes[1], 0);
1663
1664         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1665         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1666         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1667
1668         let events = nodes[0].node.get_and_clear_pending_events();
1669         assert_eq!(events.len(), 1);
1670         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1671                 assert_eq!(payment_preimage, payment_preimage_1);
1672         } else { panic!("Unexpected event!"); }
1673 }
1674
1675 #[test]
1676 fn test_monitor_update_on_pending_forwards() {
1677         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1678         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1679         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1680         // from C to A will be pending a forward to A.
1681         let chanmon_cfgs = create_chanmon_cfgs(3);
1682         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1683         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1684         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1685         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1686         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1687         let logger = test_utils::TestLogger::new();
1688
1689         // Rebalance a bit so that we can send backwards from 3 to 1.
1690         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1691
1692         let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1693         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1694         expect_pending_htlcs_forwardable!(nodes[2]);
1695         check_added_monitors!(nodes[2], 1);
1696
1697         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1698         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1699         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1700         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1701
1702         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
1703         {
1704                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1705                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1706                 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1707                 check_added_monitors!(nodes[2], 1);
1708         }
1709
1710         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1711         assert_eq!(events.len(), 1);
1712         let payment_event = SendEvent::from_event(events.pop().unwrap());
1713         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1714         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1715
1716         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1717         expect_pending_htlcs_forwardable!(nodes[1]);
1718         check_added_monitors!(nodes[1], 1);
1719         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1720         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1721
1722         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1723         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1724         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1725         check_added_monitors!(nodes[1], 0);
1726
1727         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1728         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1729         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1730         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1731
1732         let events = nodes[0].node.get_and_clear_pending_events();
1733         assert_eq!(events.len(), 2);
1734         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1735                 assert_eq!(payment_hash, payment_hash_1);
1736                 assert!(rejected_by_dest);
1737         } else { panic!("Unexpected event!"); }
1738         match events[1] {
1739                 Event::PendingHTLCsForwardable { .. } => { },
1740                 _ => panic!("Unexpected event"),
1741         };
1742         nodes[0].node.process_pending_htlc_forwards();
1743         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1744
1745         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1746 }
1747
1748 #[test]
1749 fn monitor_update_claim_fail_no_response() {
1750         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1751         // to channel being AwaitingRAA).
1752         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1753         // code was broken.
1754         let chanmon_cfgs = create_chanmon_cfgs(2);
1755         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1756         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1757         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1758         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1759         let logger = test_utils::TestLogger::new();
1760
1761         // Forward a payment for B to claim
1762         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1763
1764         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1765         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1766         {
1767                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1768                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1769                 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1770                 check_added_monitors!(nodes[0], 1);
1771         }
1772
1773         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1774         assert_eq!(events.len(), 1);
1775         let payment_event = SendEvent::from_event(events.pop().unwrap());
1776         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1777         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1778
1779         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1780         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1781         check_added_monitors!(nodes[1], 1);
1782         let events = nodes[1].node.get_and_clear_pending_msg_events();
1783         assert_eq!(events.len(), 0);
1784         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1785
1786         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1787         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1788         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1789         check_added_monitors!(nodes[1], 0);
1790         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1791
1792         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1793         check_added_monitors!(nodes[1], 1);
1794         expect_pending_htlcs_forwardable!(nodes[1]);
1795         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1796
1797         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1798         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1799         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1800
1801         let events = nodes[0].node.get_and_clear_pending_events();
1802         assert_eq!(events.len(), 1);
1803         match events[0] {
1804                 Event::PaymentSent { ref payment_preimage } => {
1805                         assert_eq!(*payment_preimage, payment_preimage_1);
1806                 },
1807                 _ => panic!("Unexpected event"),
1808         }
1809
1810         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1811 }
1812
1813 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1814 // restore_b_before_conf has no meaning if !confirm_a_first
1815 fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf: bool) {
1816         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1817         // the channel setup happily after the update is restored.
1818         let chanmon_cfgs = create_chanmon_cfgs(2);
1819         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1820         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1821         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1822
1823         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1824         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1825         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1826
1827         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1828
1829         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_tx.clone()).unwrap();
1830         check_added_monitors!(nodes[0], 0);
1831
1832         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1833         let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1834         let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1835         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1836         check_added_monitors!(nodes[1], 1);
1837
1838         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1839         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1840         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1841         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1842         check_added_monitors!(nodes[0], 1);
1843         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1844         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1845         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1846         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1847         check_added_monitors!(nodes[0], 0);
1848
1849         let events = nodes[0].node.get_and_clear_pending_events();
1850         assert_eq!(events.len(), 0);
1851         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
1852         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0)[0].txid(), funding_output.txid);
1853
1854         if confirm_a_first {
1855                 confirm_transaction(&nodes[0], &funding_tx);
1856                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1857         } else {
1858                 assert!(!restore_b_before_conf);
1859                 confirm_transaction(&nodes[1], &funding_tx);
1860                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1861         }
1862
1863         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1864         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1865         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1866         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1867         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1868         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1869
1870         if !restore_b_before_conf {
1871                 confirm_transaction(&nodes[1], &funding_tx);
1872                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1873                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1874         }
1875
1876         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1877         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1878         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1879         check_added_monitors!(nodes[1], 0);
1880
1881         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1882                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1883
1884                 confirm_transaction(&nodes[0], &funding_tx);
1885                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1886                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1887         } else {
1888                 if restore_b_before_conf {
1889                         confirm_transaction(&nodes[1], &funding_tx);
1890                 }
1891                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1892                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1893         };
1894         for node in nodes.iter() {
1895                 assert!(node.net_graph_msg_handler.handle_channel_announcement(&announcement).unwrap());
1896                 node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
1897                 node.net_graph_msg_handler.handle_channel_update(&bs_update).unwrap();
1898         }
1899
1900         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1901         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1902 }
1903
1904 #[test]
1905 fn during_funding_monitor_fail() {
1906         do_during_funding_monitor_fail(true, true);
1907         do_during_funding_monitor_fail(true, false);
1908         do_during_funding_monitor_fail(false, false);
1909 }
1910
1911 #[test]
1912 fn test_path_paused_mpp() {
1913         // Simple test of sending a multi-part payment where one path is currently blocked awaiting
1914         // monitor update
1915         let chanmon_cfgs = create_chanmon_cfgs(4);
1916         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1917         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
1918         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1919
1920         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1921         let (chan_2_ann, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known());
1922         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1923         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1924         let logger = test_utils::TestLogger::new();
1925
1926         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[3]);
1927         let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
1928
1929         // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
1930         let path = route.paths[0].clone();
1931         route.paths.push(path);
1932         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
1933         route.paths[0][0].short_channel_id = chan_1_id;
1934         route.paths[0][1].short_channel_id = chan_3_id;
1935         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
1936         route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
1937         route.paths[1][1].short_channel_id = chan_4_id;
1938
1939         // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
1940         // (for the path 0 -> 2 -> 3) fails.
1941         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1942         *nodes[0].chain_monitor.next_update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1943
1944         // Now check that we get the right return value, indicating that the first path succeeded but
1945         // the second got a MonitorUpdateFailed err. This implies PaymentSendFailure::PartialFailure as
1946         // some paths succeeded, preventing retry.
1947         if let Err(PaymentSendFailure::PartialFailure(results)) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
1948                 assert_eq!(results.len(), 2);
1949                 if let Ok(()) = results[0] {} else { panic!(); }
1950                 if let Err(APIError::MonitorUpdateFailed) = results[1] {} else { panic!(); }
1951         } else { panic!(); }
1952         check_added_monitors!(nodes[0], 2);
1953         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1954
1955         // Pass the first HTLC of the payment along to nodes[3].
1956         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1957         assert_eq!(events.len(), 1);
1958         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 0, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), false);
1959
1960         // And check that, after we successfully update the monitor for chan_2 we can pass the second
1961         // HTLC along to nodes[3] and claim the whole payment back to nodes[0].
1962         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2_id).unwrap().clone();
1963         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1964         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1965         assert_eq!(events.len(), 1);
1966         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), true);
1967
1968         claim_payment_along_route_with_secret(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage, Some(payment_secret), 200_000);
1969 }