_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
  11 //! monitor updates.
  12 //! There are a bunch of these as their handling is relatively error-prone so they are split out
  13 //! here. See also the chanmon_fail_consistency fuzz test.
  14
  15 use bitcoin::blockdata::block::{Block, BlockHeader};
  16 use bitcoin::hash_types::BlockHash;
  17 use bitcoin::network::constants::Network;
  18 use chain::channelmonitor::{ChannelMonitor, ChannelMonitorUpdateErr};
  19 use chain::transaction::OutPoint;
  20 use chain::Listen;
  21 use chain::Watch;
  22 use ln::{PaymentPreimage, PaymentHash};
  23 use ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, RAACommitmentOrder, PaymentSendFailure};
  24 use ln::features::{InitFeatures, InvoiceFeatures};
  25 use ln::msgs;
  26 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  27 use routing::router::get_route;
  28 use util::config::UserConfig;
  29 use util::enforcing_trait_impls::EnforcingSigner;
  30 use util::events::{Event, MessageSendEvent, MessageSendEventsProvider};
  31 use util::errors::APIError;
  32 use util::ser::{ReadableArgs, Writeable};
  33
  34 use bitcoin::hashes::sha256::Hash as Sha256;
  35 use bitcoin::hashes::Hash;
  36
  37 use ln::functional_test_utils::*;
  38
  39 use util::test_utils;
  40
  41 use std::collections::HashMap;
  42
  43 // If persister_fail is true, we have the persister return a PermanentFailure
  44 // instead of the higher-level ChainMonitor.
  45 fn do_test_simple_monitor_permanent_update_fail(persister_fail: bool) {
  46         // Test that we handle a simple permanent monitor update failure
  47         let mut chanmon_cfgs = create_chanmon_cfgs(2);
  48         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  49         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  50         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  51         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
  52         let logger = test_utils::TestLogger::new();
  53
  54         let (_, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(&nodes[1]);
  55
  56         match persister_fail {
  57                 true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::PermanentFailure)),
  58                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::PermanentFailure))
  59         }
  60         let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
  61         let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
  62         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), true, APIError::ChannelUnavailable {..}, {});
  63         check_added_monitors!(nodes[0], 2);
  64
  65         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  66         assert_eq!(events_1.len(), 2);
  67         match events_1[0] {
  68                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  69                 _ => panic!("Unexpected event"),
  70         };
  71         match events_1[1] {
  72                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  73                 _ => panic!("Unexpected event"),
  74         };
  75
  76         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  77         // PaymentFailed event
  78
  79         assert_eq!(nodes[0].node.list_channels().len(), 0);
  80 }
  81
  82 #[test]
  83 fn test_monitor_and_persister_update_fail() {
  84         // Test that if both updating the `ChannelMonitor` and persisting the updated
  85         // `ChannelMonitor` fail, then the failure from updating the `ChannelMonitor`
  86         // one that gets returned.
  87         let chanmon_cfgs = create_chanmon_cfgs(2);
  88         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  89         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  90         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  91
  92         // Create some initial channel
  93         let chan = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
  94         let outpoint = OutPoint { txid: chan.3.txid(), index: 0 };
  95
  96         // Rebalance the network to generate htlc in the two directions
  97         send_payment(&nodes[0], &vec!(&nodes[1])[..], 10_000_000);
  98
  99         // Route an HTLC from node 0 to node 1 (but don't settle)
 100         let preimage = route_payment(&nodes[0], &vec!(&nodes[1])[..], 9_000_000).0;
 101
 102         // Make a copy of the ChainMonitor so we can capture the error it returns on a
 103         // bogus update. Note that if instead we updated the nodes[0]'s ChainMonitor
 104         // directly, the node would fail to be `Drop`'d at the end because its
 105         // ChannelManager and ChainMonitor would be out of sync.
 106         let chain_source = test_utils::TestChainSource::new(Network::Testnet);
 107         let logger = test_utils::TestLogger::with_id(format!("node {}", 0));
 108         let persister = test_utils::TestPersister::new();
 109         let chain_mon = {
 110                 let monitors = nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap();
 111                 let monitor = monitors.get(&outpoint).unwrap();
 112                 let mut w = test_utils::TestVecWriter(Vec::new());
 113                 monitor.write(&mut w).unwrap();
 114                 let new_monitor = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
 115                         &mut ::std::io::Cursor::new(&w.0), &test_utils::OnlyReadsKeysInterface {}).unwrap().1;
 116                 assert!(new_monitor == *monitor);
 117                 let chain_mon = test_utils::TestChainMonitor::new(Some(&chain_source), &chanmon_cfgs[0].tx_broadcaster, &logger, &chanmon_cfgs[0].fee_estimator, &persister, &node_cfgs[0].keys_manager);
 118                 assert!(chain_mon.watch_channel(outpoint, new_monitor).is_ok());
 119                 chain_mon
 120         };
 121         let header = BlockHeader { version: 0x20000000, prev_blockhash: Default::default(), merkle_root: Default::default(), time: 42, bits: 42, nonce: 42 };
 122         chain_mon.chain_monitor.block_connected(&Block { header, txdata: vec![] }, 200);
 123
 124         // Set the persister's return value to be a TemporaryFailure.
 125         persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 126
 127         // Try to update ChannelMonitor
 128         assert!(nodes[1].node.claim_funds(preimage));
 129         check_added_monitors!(nodes[1], 1);
 130         let updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 131         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 132         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 133         if let Some(ref mut channel) = nodes[0].node.channel_state.lock().unwrap().by_id.get_mut(&chan.2) {
 134                 if let Ok((_, _, _, update)) = channel.commitment_signed(&updates.commitment_signed, &node_cfgs[0].fee_estimator, &node_cfgs[0].logger) {
 135                         // Check that even though the persister is returning a TemporaryFailure,
 136                         // because the update is bogus, ultimately the error that's returned
 137                         // should be a PermanentFailure.
 138                         if let Err(ChannelMonitorUpdateErr::PermanentFailure) = chain_mon.chain_monitor.update_channel(outpoint, update.clone()) {} else { panic!("Expected monitor error to be permanent"); }
 139                         logger.assert_log_contains("lightning::chain::chainmonitor".to_string(), "Failed to persist channel monitor update: TemporaryFailure".to_string(), 1);
 140                         if let Ok(_) = nodes[0].chain_monitor.update_channel(outpoint, update) {} else { assert!(false); }
 141                 } else { assert!(false); }
 142         } else { assert!(false); };
 143
 144         check_added_monitors!(nodes[0], 1);
 145         let events = nodes[0].node.get_and_clear_pending_events();
 146         assert_eq!(events.len(), 1);
 147 }
 148
 149 #[test]
 150 fn test_simple_monitor_permanent_update_fail() {
 151         do_test_simple_monitor_permanent_update_fail(false);
 152
 153         // Test behavior when the persister returns a PermanentFailure.
 154         do_test_simple_monitor_permanent_update_fail(true);
 155 }
 156
 157 // If persister_fail is true, we have the persister return a TemporaryFailure instead of the
 158 // higher-level ChainMonitor.
 159 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool, persister_fail: bool) {
 160         // Test that we can recover from a simple temporary monitor update failure optionally with
 161         // a disconnect in between
 162         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 163         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 164         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 165         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 166         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 167         let logger = test_utils::TestLogger::new();
 168
 169         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(&nodes[1]);
 170
 171         match persister_fail {
 172                 true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure)),
 173                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure))
 174         }
 175
 176         {
 177                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 178                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 179                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)), false, APIError::MonitorUpdateFailed, {});
 180                 check_added_monitors!(nodes[0], 1);
 181         }
 182
 183         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 184         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 185         assert_eq!(nodes[0].node.list_channels().len(), 1);
 186
 187         if disconnect {
 188                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 189                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 190                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 191         }
 192
 193         match persister_fail {
 194                 true => chanmon_cfgs[0].persister.set_update_ret(Ok(())),
 195                 false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()))
 196         }
 197         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 198         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 199         check_added_monitors!(nodes[0], 0);
 200
 201         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
 202         assert_eq!(events_2.len(), 1);
 203         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
 204         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 205         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 206         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 207
 208         expect_pending_htlcs_forwardable!(nodes[1]);
 209
 210         let events_3 = nodes[1].node.get_and_clear_pending_events();
 211         assert_eq!(events_3.len(), 1);
 212         match events_3[0] {
 213                 Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
 214                         assert_eq!(payment_hash_1, *payment_hash);
 215                         assert!(payment_preimage.is_none());
 216                         assert_eq!(payment_secret_1, *payment_secret);
 217                         assert_eq!(amt, 1000000);
 218                 },
 219                 _ => panic!("Unexpected event"),
 220         }
 221
 222         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 223
 224         // Now set it to failed again...
 225         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(&nodes[1]);
 226         {
 227                 match persister_fail {
 228                         true => chanmon_cfgs[0].persister.set_update_ret(Err(ChannelMonitorUpdateErr::TemporaryFailure)),
 229                         false => *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure))
 230                 }
 231                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 232                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 233                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateFailed, {});
 234                 check_added_monitors!(nodes[0], 1);
 235         }
 236
 237         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 238         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 239         assert_eq!(nodes[0].node.list_channels().len(), 1);
 240
 241         if disconnect {
 242                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 243                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 244                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 245         }
 246
 247         // ...and make sure we can force-close a frozen channel
 248         nodes[0].node.force_close_channel(&channel_id).unwrap();
 249         check_added_monitors!(nodes[0], 1);
 250         check_closed_broadcast!(nodes[0], true);
 251
 252         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 253         // PaymentFailed event
 254
 255         assert_eq!(nodes[0].node.list_channels().len(), 0);
 256 }
 257
 258 #[test]
 259 fn test_simple_monitor_temporary_update_fail() {
 260         do_test_simple_monitor_temporary_update_fail(false, false);
 261         do_test_simple_monitor_temporary_update_fail(true, false);
 262
 263         // Test behavior when the persister returns a TemporaryFailure.
 264         do_test_simple_monitor_temporary_update_fail(false, true);
 265         do_test_simple_monitor_temporary_update_fail(true, true);
 266 }
 267
 268 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 269         let disconnect_flags = 8 | 16;
 270
 271         // Test that we can recover from a temporary monitor update failure with some in-flight
 272         // HTLCs going on at the same time potentially with some disconnection thrown in.
 273         // * First we route a payment, then get a temporary monitor update failure when trying to
 274         //   route a second payment. We then claim the first payment.
 275         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 276         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 277         //   the ChannelMonitor on a watchtower).
 278         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 279         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 280         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 281         //   disconnect_count & !disconnect_flags is 0).
 282         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 283         //   through message sending, potentially disconnect/reconnecting multiple times based on
 284         //   disconnect_count, to get the update_fulfill_htlc through.
 285         // * We then walk through more message exchanges to get the original update_add_htlc
 286         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 287         //   disconnect/reconnecting based on disconnect_count.
 288         let chanmon_cfgs = create_chanmon_cfgs(2);
 289         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 290         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 291         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 292         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 293         let logger = test_utils::TestLogger::new();
 294
 295         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 296
 297         // Now try to send a second payment which will fail to send
 298         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
 299         {
 300                 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 301                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 302                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 303                 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)), false, APIError::MonitorUpdateFailed, {});
 304                 check_added_monitors!(nodes[0], 1);
 305         }
 306
 307         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 308         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 309         assert_eq!(nodes[0].node.list_channels().len(), 1);
 310
 311         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 312         // but nodes[0] won't respond since it is frozen.
 313         assert!(nodes[1].node.claim_funds(payment_preimage_1));
 314         check_added_monitors!(nodes[1], 1);
 315         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 316         assert_eq!(events_2.len(), 1);
 317         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 318                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 319                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 320                         assert!(update_add_htlcs.is_empty());
 321                         assert_eq!(update_fulfill_htlcs.len(), 1);
 322                         assert!(update_fail_htlcs.is_empty());
 323                         assert!(update_fail_malformed_htlcs.is_empty());
 324                         assert!(update_fee.is_none());
 325
 326                         if (disconnect_count & 16) == 0 {
 327                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 328                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 329                                 assert_eq!(events_3.len(), 1);
 330                                 match events_3[0] {
 331                                         Event::PaymentSent { ref payment_preimage } => {
 332                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 333                                         },
 334                                         _ => panic!("Unexpected event"),
 335                                 }
 336
 337                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 338                                 check_added_monitors!(nodes[0], 1);
 339                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 340                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 341                         }
 342
 343                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 344                 },
 345                 _ => panic!("Unexpected event"),
 346         };
 347
 348         if disconnect_count & !disconnect_flags > 0 {
 349                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 350                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 351         }
 352
 353         // Now fix monitor updating...
 354         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 355         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 356         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 357         check_added_monitors!(nodes[0], 0);
 358
 359         macro_rules! disconnect_reconnect_peers { () => { {
 360                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 361                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 362
 363                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 364                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 365                 assert_eq!(reestablish_1.len(), 1);
 366                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 367                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 368                 assert_eq!(reestablish_2.len(), 1);
 369
 370                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 371                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 372                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 373                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 374
 375                 assert!(as_resp.0.is_none());
 376                 assert!(bs_resp.0.is_none());
 377
 378                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 379         } } }
 380
 381         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 382                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 383                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 384
 385                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 386                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 387                 assert_eq!(reestablish_1.len(), 1);
 388                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 389                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 390                 assert_eq!(reestablish_2.len(), 1);
 391
 392                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 393                 check_added_monitors!(nodes[0], 0);
 394                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 395                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 396                 check_added_monitors!(nodes[1], 0);
 397                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 398
 399                 assert!(as_resp.0.is_none());
 400                 assert!(bs_resp.0.is_none());
 401
 402                 assert!(bs_resp.1.is_none());
 403                 if (disconnect_count & 16) == 0 {
 404                         assert!(bs_resp.2.is_none());
 405
 406                         assert!(as_resp.1.is_some());
 407                         assert!(as_resp.2.is_some());
 408                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 409                 } else {
 410                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 411                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 412                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 413                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 414                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 415                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 416
 417                         assert!(as_resp.1.is_none());
 418
 419                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 420                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 421                         assert_eq!(events_3.len(), 1);
 422                         match events_3[0] {
 423                                 Event::PaymentSent { ref payment_preimage } => {
 424                                         assert_eq!(*payment_preimage, payment_preimage_1);
 425                                 },
 426                                 _ => panic!("Unexpected event"),
 427                         }
 428
 429                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 430                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 431                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 432                         check_added_monitors!(nodes[0], 1);
 433
 434                         as_resp.1 = Some(as_resp_raa);
 435                         bs_resp.2 = None;
 436                 }
 437
 438                 if disconnect_count & !disconnect_flags > 1 {
 439                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 440
 441                         if (disconnect_count & 16) == 0 {
 442                                 assert!(reestablish_1 == second_reestablish_1);
 443                                 assert!(reestablish_2 == second_reestablish_2);
 444                         }
 445                         assert!(as_resp == second_as_resp);
 446                         assert!(bs_resp == second_bs_resp);
 447                 }
 448
 449                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 450         } else {
 451                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 452                 assert_eq!(events_4.len(), 2);
 453                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 454                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 455                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 456                                 msg.clone()
 457                         },
 458                         _ => panic!("Unexpected event"),
 459                 })
 460         };
 461
 462         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 463
 464         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 465         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 466         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 467         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 468         check_added_monitors!(nodes[1], 1);
 469
 470         if disconnect_count & !disconnect_flags > 2 {
 471                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 472
 473                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 474                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 475
 476                 assert!(as_resp.2.is_none());
 477                 assert!(bs_resp.2.is_none());
 478         }
 479
 480         let as_commitment_update;
 481         let bs_second_commitment_update;
 482
 483         macro_rules! handle_bs_raa { () => {
 484                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 485                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 486                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 487                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 488                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 489                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 490                 assert!(as_commitment_update.update_fee.is_none());
 491                 check_added_monitors!(nodes[0], 1);
 492         } }
 493
 494         macro_rules! handle_initial_raa { () => {
 495                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 496                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 497                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 498                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 499                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 500                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 501                 assert!(bs_second_commitment_update.update_fee.is_none());
 502                 check_added_monitors!(nodes[1], 1);
 503         } }
 504
 505         if (disconnect_count & 8) == 0 {
 506                 handle_bs_raa!();
 507
 508                 if disconnect_count & !disconnect_flags > 3 {
 509                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 510
 511                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 512                         assert!(bs_resp.1.is_none());
 513
 514                         assert!(as_resp.2.unwrap() == as_commitment_update);
 515                         assert!(bs_resp.2.is_none());
 516
 517                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 518                 }
 519
 520                 handle_initial_raa!();
 521
 522                 if disconnect_count & !disconnect_flags > 4 {
 523                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 524
 525                         assert!(as_resp.1.is_none());
 526                         assert!(bs_resp.1.is_none());
 527
 528                         assert!(as_resp.2.unwrap() == as_commitment_update);
 529                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 530                 }
 531         } else {
 532                 handle_initial_raa!();
 533
 534                 if disconnect_count & !disconnect_flags > 3 {
 535                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 536
 537                         assert!(as_resp.1.is_none());
 538                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 539
 540                         assert!(as_resp.2.is_none());
 541                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 542
 543                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 544                 }
 545
 546                 handle_bs_raa!();
 547
 548                 if disconnect_count & !disconnect_flags > 4 {
 549                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 550
 551                         assert!(as_resp.1.is_none());
 552                         assert!(bs_resp.1.is_none());
 553
 554                         assert!(as_resp.2.unwrap() == as_commitment_update);
 555                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 556                 }
 557         }
 558
 559         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 560         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 561         // No commitment_signed so get_event_msg's assert(len == 1) passes
 562         check_added_monitors!(nodes[0], 1);
 563
 564         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 565         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 566         // No commitment_signed so get_event_msg's assert(len == 1) passes
 567         check_added_monitors!(nodes[1], 1);
 568
 569         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 570         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 571         check_added_monitors!(nodes[1], 1);
 572
 573         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 574         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 575         check_added_monitors!(nodes[0], 1);
 576
 577         expect_pending_htlcs_forwardable!(nodes[1]);
 578
 579         let events_5 = nodes[1].node.get_and_clear_pending_events();
 580         assert_eq!(events_5.len(), 1);
 581         match events_5[0] {
 582                 Event::PaymentReceived { ref payment_hash, ref payment_preimage, ref payment_secret, amt, user_payment_id: _ } => {
 583                         assert_eq!(payment_hash_2, *payment_hash);
 584                         assert!(payment_preimage.is_none());
 585                         assert_eq!(payment_secret_2, *payment_secret);
 586                         assert_eq!(amt, 1000000);
 587                 },
 588                 _ => panic!("Unexpected event"),
 589         }
 590
 591         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
 592 }
 593
 594 #[test]
 595 fn test_monitor_temporary_update_fail_a() {
 596         do_test_monitor_temporary_update_fail(0);
 597         do_test_monitor_temporary_update_fail(1);
 598         do_test_monitor_temporary_update_fail(2);
 599         do_test_monitor_temporary_update_fail(3);
 600         do_test_monitor_temporary_update_fail(4);
 601         do_test_monitor_temporary_update_fail(5);
 602 }
 603
 604 #[test]
 605 fn test_monitor_temporary_update_fail_b() {
 606         do_test_monitor_temporary_update_fail(2 | 8);
 607         do_test_monitor_temporary_update_fail(3 | 8);
 608         do_test_monitor_temporary_update_fail(4 | 8);
 609         do_test_monitor_temporary_update_fail(5 | 8);
 610 }
 611
 612 #[test]
 613 fn test_monitor_temporary_update_fail_c() {
 614         do_test_monitor_temporary_update_fail(1 | 16);
 615         do_test_monitor_temporary_update_fail(2 | 16);
 616         do_test_monitor_temporary_update_fail(3 | 16);
 617         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 618         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 619 }
 620
 621 #[test]
 622 fn test_monitor_update_fail_cs() {
 623         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 624         let chanmon_cfgs = create_chanmon_cfgs(2);
 625         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 626         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 627         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 628         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 629         let logger = test_utils::TestLogger::new();
 630
 631         let (payment_preimage, our_payment_hash, our_payment_secret) = get_payment_preimage_hash!(nodes[1]);
 632         {
 633                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 634                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 635                 nodes[0].node.send_payment(&route, our_payment_hash, &Some(our_payment_secret)).unwrap();
 636                 check_added_monitors!(nodes[0], 1);
 637         }
 638
 639         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 640         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 641
 642         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 643         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 644         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 645         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 646         check_added_monitors!(nodes[1], 1);
 647         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 648
 649         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 650         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 651         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 652         check_added_monitors!(nodes[1], 0);
 653         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 654         assert_eq!(responses.len(), 2);
 655
 656         match responses[0] {
 657                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 658                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 659                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 660                         check_added_monitors!(nodes[0], 1);
 661                 },
 662                 _ => panic!("Unexpected event"),
 663         }
 664         match responses[1] {
 665                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 666                         assert!(updates.update_add_htlcs.is_empty());
 667                         assert!(updates.update_fulfill_htlcs.is_empty());
 668                         assert!(updates.update_fail_htlcs.is_empty());
 669                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 670                         assert!(updates.update_fee.is_none());
 671                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 672
 673                         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 674                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 675                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 676                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 677                         check_added_monitors!(nodes[0], 1);
 678                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 679                 },
 680                 _ => panic!("Unexpected event"),
 681         }
 682
 683         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 684         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 685         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 686         check_added_monitors!(nodes[0], 0);
 687
 688         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 689         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 690         check_added_monitors!(nodes[1], 1);
 691
 692         expect_pending_htlcs_forwardable!(nodes[1]);
 693
 694         let events = nodes[1].node.get_and_clear_pending_events();
 695         assert_eq!(events.len(), 1);
 696         match events[0] {
 697                 Event::PaymentReceived { payment_hash, payment_preimage, payment_secret, amt, user_payment_id: _ } => {
 698                         assert_eq!(payment_hash, our_payment_hash);
 699                         assert!(payment_preimage.is_none());
 700                         assert_eq!(our_payment_secret, payment_secret);
 701                         assert_eq!(amt, 1000000);
 702                 },
 703                 _ => panic!("Unexpected event"),
 704         };
 705
 706         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 707 }
 708
 709 #[test]
 710 fn test_monitor_update_fail_no_rebroadcast() {
 711         // Tests handling of a monitor update failure when no message rebroadcasting on
 712         // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
 713         // fuzz tests.
 714         let chanmon_cfgs = create_chanmon_cfgs(2);
 715         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 716         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 717         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 718         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 719         let logger = test_utils::TestLogger::new();
 720
 721         let (payment_preimage_1, our_payment_hash, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
 722         {
 723                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 724                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 725                 nodes[0].node.send_payment(&route, our_payment_hash, &Some(payment_secret_1)).unwrap();
 726                 check_added_monitors!(nodes[0], 1);
 727         }
 728
 729         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 730         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 731         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 732
 733         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 734         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 735         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 736         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 737         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 738         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 739         check_added_monitors!(nodes[1], 1);
 740
 741         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 742         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 743         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 744         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 745         check_added_monitors!(nodes[1], 0);
 746         expect_pending_htlcs_forwardable!(nodes[1]);
 747
 748         let events = nodes[1].node.get_and_clear_pending_events();
 749         assert_eq!(events.len(), 1);
 750         match events[0] {
 751                 Event::PaymentReceived { payment_hash, .. } => {
 752                         assert_eq!(payment_hash, our_payment_hash);
 753                 },
 754                 _ => panic!("Unexpected event"),
 755         }
 756
 757         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 758 }
 759
 760 #[test]
 761 fn test_monitor_update_raa_while_paused() {
 762         // Tests handling of an RAA while monitor updating has already been marked failed.
 763         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 764         let chanmon_cfgs = create_chanmon_cfgs(2);
 765         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 766         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 767         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 768         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 769         let logger = test_utils::TestLogger::new();
 770
 771         send_payment(&nodes[0], &[&nodes[1]], 5000000);
 772         let (payment_preimage_1, our_payment_hash_1, our_payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
 773         {
 774                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 775                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 776                 nodes[0].node.send_payment(&route, our_payment_hash_1, &Some(our_payment_secret_1)).unwrap();
 777                 check_added_monitors!(nodes[0], 1);
 778         }
 779         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 780
 781         let (payment_preimage_2, our_payment_hash_2, our_payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
 782         {
 783                 let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
 784                 let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 785                 nodes[1].node.send_payment(&route, our_payment_hash_2, &Some(our_payment_secret_2)).unwrap();
 786                 check_added_monitors!(nodes[1], 1);
 787         }
 788         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 789
 790         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 791         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 792         check_added_monitors!(nodes[1], 1);
 793         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 794
 795         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 796         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 797         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 798         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 799         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 800         check_added_monitors!(nodes[0], 1);
 801
 802         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 803         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 804         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 805         check_added_monitors!(nodes[0], 1);
 806
 807         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 808         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 809         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 810         check_added_monitors!(nodes[0], 0);
 811
 812         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 813         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 814         check_added_monitors!(nodes[1], 1);
 815         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 816
 817         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 818         check_added_monitors!(nodes[1], 1);
 819         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 820
 821         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 822         check_added_monitors!(nodes[0], 1);
 823         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 824
 825         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 826         check_added_monitors!(nodes[0], 1);
 827         expect_pending_htlcs_forwardable!(nodes[0]);
 828         expect_payment_received!(nodes[0], our_payment_hash_2, our_payment_secret_2, 1000000);
 829
 830         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 831         check_added_monitors!(nodes[1], 1);
 832         expect_pending_htlcs_forwardable!(nodes[1]);
 833         expect_payment_received!(nodes[1], our_payment_hash_1, our_payment_secret_1, 1000000);
 834
 835         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 836         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2);
 837 }
 838
 839 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 840         // Tests handling of a monitor update failure when processing an incoming RAA
 841         let chanmon_cfgs = create_chanmon_cfgs(3);
 842         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 843         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 844         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 845         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
 846         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
 847         let logger = test_utils::TestLogger::new();
 848
 849         // Rebalance a bit so that we can send backwards from 2 to 1.
 850         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000);
 851
 852         // Route a first payment that we'll fail backwards
 853         let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 854
 855         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 856         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 857         expect_pending_htlcs_forwardable!(nodes[2]);
 858         check_added_monitors!(nodes[2], 1);
 859
 860         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 861         assert!(updates.update_add_htlcs.is_empty());
 862         assert!(updates.update_fulfill_htlcs.is_empty());
 863         assert_eq!(updates.update_fail_htlcs.len(), 1);
 864         assert!(updates.update_fail_malformed_htlcs.is_empty());
 865         assert!(updates.update_fee.is_none());
 866         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 867
 868         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 869         check_added_monitors!(nodes[0], 0);
 870
 871         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 872         // holding cell.
 873         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[2]);
 874         {
 875                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 876                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 877                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
 878                 check_added_monitors!(nodes[0], 1);
 879         }
 880
 881         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 882         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 883         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 884
 885         expect_pending_htlcs_forwardable!(nodes[1]);
 886         check_added_monitors!(nodes[1], 0);
 887         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 888
 889         // Now fail monitor updating.
 890         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
 891         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 892         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 893         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 894         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 895         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 896         check_added_monitors!(nodes[1], 1);
 897
 898         // Attempt to forward a third payment but fail due to the second channel being unavailable
 899         // for forwarding.
 900         let (_, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[2]);
 901         {
 902                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
 903                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 904                 nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
 905                 check_added_monitors!(nodes[0], 1);
 906         }
 907
 908         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(())); // We succeed in updating the monitor for the first channel
 909         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 910         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 911         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 912         check_added_monitors!(nodes[1], 0);
 913
 914         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 915         assert_eq!(events_2.len(), 1);
 916         match events_2.remove(0) {
 917                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 918                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 919                         assert!(updates.update_fulfill_htlcs.is_empty());
 920                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 921                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 922                         assert!(updates.update_add_htlcs.is_empty());
 923                         assert!(updates.update_fee.is_none());
 924
 925                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 926                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 927
 928                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 929                         assert_eq!(msg_events.len(), 1);
 930                         match msg_events[0] {
 931                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 932                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 933                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 934                                 },
 935                                 _ => panic!("Unexpected event"),
 936                         }
 937
 938                         let events = nodes[0].node.get_and_clear_pending_events();
 939                         assert_eq!(events.len(), 1);
 940                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 941                                 assert_eq!(payment_hash, payment_hash_3);
 942                                 assert!(!rejected_by_dest);
 943                         } else { panic!("Unexpected event!"); }
 944                 },
 945                 _ => panic!("Unexpected event type!"),
 946         };
 947
 948         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 949                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 950                 let (payment_preimage_4, payment_hash_4, payment_secret_4) = get_payment_preimage_hash!(nodes[0]);
 951                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
 952                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
 953                 nodes[2].node.send_payment(&route, payment_hash_4, &Some(payment_secret_4)).unwrap();
 954                 check_added_monitors!(nodes[2], 1);
 955
 956                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 957                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 958                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 959                 check_added_monitors!(nodes[1], 1);
 960                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 961                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 962                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 963                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 964                 (Some(payment_preimage_4), Some(payment_hash_4))
 965         } else { (None, None) };
 966
 967         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 968         // update_add update.
 969         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
 970         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
 971         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 972         check_added_monitors!(nodes[1], 0);
 973         expect_pending_htlcs_forwardable!(nodes[1]);
 974         check_added_monitors!(nodes[1], 1);
 975
 976         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 977         if test_ignore_second_cs {
 978                 assert_eq!(events_3.len(), 3);
 979         } else {
 980                 assert_eq!(events_3.len(), 2);
 981         }
 982
 983         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 984         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 985         let messages_a = match events_3.pop().unwrap() {
 986                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 987                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 988                         assert!(updates.update_fulfill_htlcs.is_empty());
 989                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 990                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 991                         assert!(updates.update_add_htlcs.is_empty());
 992                         assert!(updates.update_fee.is_none());
 993                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 994                 },
 995                 _ => panic!("Unexpected event type!"),
 996         };
 997         let raa = if test_ignore_second_cs {
 998                 match events_3.remove(1) {
 999                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
1000                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
1001                                 Some(msg.clone())
1002                         },
1003                         _ => panic!("Unexpected event"),
1004                 }
1005         } else { None };
1006         let send_event_b = SendEvent::from_event(events_3.remove(0));
1007         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
1008
1009         // Now deliver the new messages...
1010
1011         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
1012         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
1013         let events_4 = nodes[0].node.get_and_clear_pending_events();
1014         assert_eq!(events_4.len(), 1);
1015         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
1016                 assert_eq!(payment_hash, payment_hash_1);
1017                 assert!(rejected_by_dest);
1018         } else { panic!("Unexpected event!"); }
1019
1020         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
1021         if test_ignore_second_cs {
1022                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
1023                 check_added_monitors!(nodes[2], 1);
1024                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1025                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
1026                 check_added_monitors!(nodes[2], 1);
1027                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1028                 assert!(bs_cs.update_add_htlcs.is_empty());
1029                 assert!(bs_cs.update_fail_htlcs.is_empty());
1030                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
1031                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
1032                 assert!(bs_cs.update_fee.is_none());
1033
1034                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
1035                 check_added_monitors!(nodes[1], 1);
1036                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1037                 assert!(as_cs.update_add_htlcs.is_empty());
1038                 assert!(as_cs.update_fail_htlcs.is_empty());
1039                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
1040                 assert!(as_cs.update_fulfill_htlcs.is_empty());
1041                 assert!(as_cs.update_fee.is_none());
1042
1043                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
1044                 check_added_monitors!(nodes[1], 1);
1045                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
1046
1047                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
1048                 check_added_monitors!(nodes[2], 1);
1049                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1050
1051                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
1052                 check_added_monitors!(nodes[2], 1);
1053                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
1054
1055                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
1056                 check_added_monitors!(nodes[1], 1);
1057                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1058         } else {
1059                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
1060         }
1061
1062         expect_pending_htlcs_forwardable!(nodes[2]);
1063
1064         let events_6 = nodes[2].node.get_and_clear_pending_events();
1065         assert_eq!(events_6.len(), 1);
1066         match events_6[0] {
1067                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
1068                 _ => panic!("Unexpected event"),
1069         };
1070
1071         if test_ignore_second_cs {
1072                 expect_pending_htlcs_forwardable!(nodes[1]);
1073                 check_added_monitors!(nodes[1], 1);
1074
1075                 send_event = SendEvent::from_node(&nodes[1]);
1076                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
1077                 assert_eq!(send_event.msgs.len(), 1);
1078                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
1079                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
1080
1081                 expect_pending_htlcs_forwardable!(nodes[0]);
1082
1083                 let events_9 = nodes[0].node.get_and_clear_pending_events();
1084                 assert_eq!(events_9.len(), 1);
1085                 match events_9[0] {
1086                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
1087                         _ => panic!("Unexpected event"),
1088                 };
1089                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap());
1090         }
1091
1092         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
1093 }
1094
1095 #[test]
1096 fn test_monitor_update_fail_raa() {
1097         do_test_monitor_update_fail_raa(false);
1098         do_test_monitor_update_fail_raa(true);
1099 }
1100
1101 #[test]
1102 fn test_monitor_update_fail_reestablish() {
1103         // Simple test for message retransmission after monitor update failure on
1104         // channel_reestablish generating a monitor update (which comes from freeing holding cell
1105         // HTLCs).
1106         let chanmon_cfgs = create_chanmon_cfgs(3);
1107         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1108         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1109         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1110         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1111         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1112
1113         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1114
1115         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1116         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1117
1118         assert!(nodes[2].node.claim_funds(our_payment_preimage));
1119         check_added_monitors!(nodes[2], 1);
1120         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1121         assert!(updates.update_add_htlcs.is_empty());
1122         assert!(updates.update_fail_htlcs.is_empty());
1123         assert!(updates.update_fail_malformed_htlcs.is_empty());
1124         assert!(updates.update_fee.is_none());
1125         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1126         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1127         check_added_monitors!(nodes[1], 1);
1128         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1129         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
1130
1131         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1132         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1133         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1134
1135         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1136         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1137
1138         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1139
1140         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1141         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1142         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1143         check_added_monitors!(nodes[1], 1);
1144
1145         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1146         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1147
1148         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1149         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1150
1151         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
1152         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
1153
1154         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1155
1156         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1157         check_added_monitors!(nodes[1], 0);
1158         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1159
1160         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1161         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1162         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1163         check_added_monitors!(nodes[1], 0);
1164
1165         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1166         assert!(updates.update_add_htlcs.is_empty());
1167         assert!(updates.update_fail_htlcs.is_empty());
1168         assert!(updates.update_fail_malformed_htlcs.is_empty());
1169         assert!(updates.update_fee.is_none());
1170         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1171         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1172         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1173
1174         let events = nodes[0].node.get_and_clear_pending_events();
1175         assert_eq!(events.len(), 1);
1176         match events[0] {
1177                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1178                 _ => panic!("Unexpected event"),
1179         }
1180 }
1181
1182 #[test]
1183 fn raa_no_response_awaiting_raa_state() {
1184         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1185         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1186         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1187         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1188         let chanmon_cfgs = create_chanmon_cfgs(2);
1189         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1190         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1191         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1192         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1193         let logger = test_utils::TestLogger::new();
1194
1195         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1196         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1197         let (payment_preimage_3, payment_hash_3, payment_secret_3) = get_payment_preimage_hash!(nodes[1]);
1198
1199         // Queue up two payments - one will be delivered right away, one immediately goes into the
1200         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1201         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1202         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1203         // generation during RAA while in monitor-update-failed state.
1204         {
1205                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1206                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1207                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
1208                 check_added_monitors!(nodes[0], 1);
1209                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1210                 check_added_monitors!(nodes[0], 0);
1211         }
1212
1213         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1214         assert_eq!(events.len(), 1);
1215         let payment_event = SendEvent::from_event(events.pop().unwrap());
1216         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1217         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1218         check_added_monitors!(nodes[1], 1);
1219
1220         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1221         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1222         check_added_monitors!(nodes[0], 1);
1223         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1224         assert_eq!(events.len(), 1);
1225         let payment_event = SendEvent::from_event(events.pop().unwrap());
1226
1227         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1228         check_added_monitors!(nodes[0], 1);
1229         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1230
1231         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1232         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1233         // then restore channel monitor updates.
1234         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1235         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1236         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1237         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1238         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1239         check_added_monitors!(nodes[1], 1);
1240
1241         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1242         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1243         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1244         check_added_monitors!(nodes[1], 1);
1245
1246         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1247         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1248         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1249         // nodes[1] should be AwaitingRAA here!
1250         check_added_monitors!(nodes[1], 0);
1251         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1252         expect_pending_htlcs_forwardable!(nodes[1]);
1253         expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
1254
1255         // We send a third payment here, which is somewhat of a redundant test, but the
1256         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1257         // commitment transaction states) whereas here we can explicitly check for it.
1258         {
1259                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1260                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1261                 nodes[0].node.send_payment(&route, payment_hash_3, &Some(payment_secret_3)).unwrap();
1262                 check_added_monitors!(nodes[0], 0);
1263                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1264         }
1265         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1266         check_added_monitors!(nodes[0], 1);
1267         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1268         assert_eq!(events.len(), 1);
1269         let payment_event = SendEvent::from_event(events.pop().unwrap());
1270
1271         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1272         check_added_monitors!(nodes[0], 1);
1273         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1274
1275         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1276         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1277         check_added_monitors!(nodes[1], 1);
1278         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1279
1280         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1281         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1282         check_added_monitors!(nodes[1], 1);
1283         expect_pending_htlcs_forwardable!(nodes[1]);
1284         expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
1285         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1286
1287         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1288         check_added_monitors!(nodes[0], 1);
1289
1290         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1291         check_added_monitors!(nodes[0], 1);
1292         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1293
1294         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1295         check_added_monitors!(nodes[1], 1);
1296         expect_pending_htlcs_forwardable!(nodes[1]);
1297         expect_payment_received!(nodes[1], payment_hash_3, payment_secret_3, 1000000);
1298
1299         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
1300         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
1301         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3);
1302 }
1303
1304 #[test]
1305 fn claim_while_disconnected_monitor_update_fail() {
1306         // Test for claiming a payment while disconnected and then having the resulting
1307         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1308         // contrived case for nodes with network instability.
1309         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1310         // code introduced a regression in this test (specifically, this caught a removal of the
1311         // channel_reestablish handling ensuring the order was sensical given the messages used).
1312         let chanmon_cfgs = create_chanmon_cfgs(2);
1313         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1314         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1315         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1316         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1317         let logger = test_utils::TestLogger::new();
1318
1319         // Forward a payment for B to claim
1320         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1321
1322         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1323         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1324
1325         assert!(nodes[1].node.claim_funds(payment_preimage_1));
1326         check_added_monitors!(nodes[1], 1);
1327
1328         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1329         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1330
1331         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1332         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1333
1334         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1335         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1336
1337         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1338         // update.
1339         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1340
1341         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1342         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1343         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1344         check_added_monitors!(nodes[1], 1);
1345         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1346
1347         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1348         // the monitor still failed
1349         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1350         {
1351                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1352                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1353                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1354                 check_added_monitors!(nodes[0], 1);
1355         }
1356
1357         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1358         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1359         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1360         check_added_monitors!(nodes[1], 1);
1361         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1362         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1363         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1364         // until we've channel_monitor_update'd and updated for the new commitment transaction.
1365
1366         // Now un-fail the monitor, which will result in B sending its original commitment update,
1367         // receiving the commitment update from A, and the resulting commitment dances.
1368         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1369         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1370         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1371         check_added_monitors!(nodes[1], 0);
1372
1373         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1374         assert_eq!(bs_msgs.len(), 2);
1375
1376         match bs_msgs[0] {
1377                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1378                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1379                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1380                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1381                         check_added_monitors!(nodes[0], 1);
1382
1383                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1384                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1385                         check_added_monitors!(nodes[1], 1);
1386                 },
1387                 _ => panic!("Unexpected event"),
1388         }
1389
1390         match bs_msgs[1] {
1391                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1392                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1393                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1394                         check_added_monitors!(nodes[0], 1);
1395                 },
1396                 _ => panic!("Unexpected event"),
1397         }
1398
1399         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1400
1401         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1402         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1403         check_added_monitors!(nodes[0], 1);
1404         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1405
1406         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1407         check_added_monitors!(nodes[1], 1);
1408         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1409         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1410         check_added_monitors!(nodes[1], 1);
1411
1412         expect_pending_htlcs_forwardable!(nodes[1]);
1413         expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
1414
1415         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1416         check_added_monitors!(nodes[0], 1);
1417
1418         let events = nodes[0].node.get_and_clear_pending_events();
1419         assert_eq!(events.len(), 1);
1420         match events[0] {
1421                 Event::PaymentSent { ref payment_preimage } => {
1422                         assert_eq!(*payment_preimage, payment_preimage_1);
1423                 },
1424                 _ => panic!("Unexpected event"),
1425         }
1426
1427         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
1428 }
1429
1430 #[test]
1431 fn monitor_failed_no_reestablish_response() {
1432         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1433         // response to a commitment_signed.
1434         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1435         // debug_assert!() failure in channel_reestablish handling.
1436         let chanmon_cfgs = create_chanmon_cfgs(2);
1437         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1438         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1439         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1440         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1441         let logger = test_utils::TestLogger::new();
1442
1443         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1444         // on receipt).
1445         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1446         {
1447                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1448                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1449                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
1450                 check_added_monitors!(nodes[0], 1);
1451         }
1452
1453         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1454         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1455         assert_eq!(events.len(), 1);
1456         let payment_event = SendEvent::from_event(events.pop().unwrap());
1457         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1458         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1459         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1460         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1461         check_added_monitors!(nodes[1], 1);
1462
1463         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1464         // is still failing to update monitors.
1465         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1466         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1467
1468         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1469         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1470
1471         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1472         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1473
1474         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1475         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1476
1477         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1478         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1479         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1480         check_added_monitors!(nodes[1], 0);
1481         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1482
1483         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1484         check_added_monitors!(nodes[0], 1);
1485         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1486         check_added_monitors!(nodes[0], 1);
1487
1488         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1489         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1490         check_added_monitors!(nodes[1], 1);
1491
1492         expect_pending_htlcs_forwardable!(nodes[1]);
1493         expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
1494
1495         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
1496 }
1497
1498 #[test]
1499 fn first_message_on_recv_ordering() {
1500         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1501         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1502         // a commitment_signed which needs to send an RAA first.
1503         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1504         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1505         // response. To do this, we start routing two payments, with the final RAA for the first being
1506         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1507         // have no pending response but will want to send a RAA/CS (with the updates for the second
1508         // payment applied).
1509         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1510         let chanmon_cfgs = create_chanmon_cfgs(2);
1511         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1512         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1513         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1514         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1515         let logger = test_utils::TestLogger::new();
1516
1517         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1518         // can deliver it and fail the monitor update.
1519         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(nodes[1]);
1520         {
1521                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1522                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1523                 nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
1524                 check_added_monitors!(nodes[0], 1);
1525         }
1526
1527         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1528         assert_eq!(events.len(), 1);
1529         let payment_event = SendEvent::from_event(events.pop().unwrap());
1530         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1531         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1532         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1533         check_added_monitors!(nodes[1], 1);
1534         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1535
1536         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1537         check_added_monitors!(nodes[0], 1);
1538         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1539         check_added_monitors!(nodes[0], 1);
1540
1541         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1542
1543         // Route the second payment, generating an update_add_htlc/commitment_signed
1544         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1545         {
1546                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1547                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1548                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1549                 check_added_monitors!(nodes[0], 1);
1550         }
1551         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1552         assert_eq!(events.len(), 1);
1553         let payment_event = SendEvent::from_event(events.pop().unwrap());
1554         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1555
1556         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1557
1558         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1559         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1560         // to the next message also tests resetting the delivery order.
1561         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1562         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1563         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1564         check_added_monitors!(nodes[1], 1);
1565
1566         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1567         // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1568         // appropriate HTLC acceptance).
1569         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1570         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1571         check_added_monitors!(nodes[1], 1);
1572         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1573         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1574
1575         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1576         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1577         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1578         check_added_monitors!(nodes[1], 0);
1579
1580         expect_pending_htlcs_forwardable!(nodes[1]);
1581         expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 1000000);
1582
1583         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1584         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1585         check_added_monitors!(nodes[0], 1);
1586         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1587         check_added_monitors!(nodes[0], 1);
1588
1589         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1590         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1591         check_added_monitors!(nodes[1], 1);
1592
1593         expect_pending_htlcs_forwardable!(nodes[1]);
1594         expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
1595
1596         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
1597         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
1598 }
1599
1600 #[test]
1601 fn test_monitor_update_fail_claim() {
1602         // Basic test for monitor update failures when processing claim_funds calls.
1603         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1604         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1605         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1606         // updating and claim the payment on B.
1607         let chanmon_cfgs = create_chanmon_cfgs(3);
1608         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1609         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1610         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1611         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1612         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1613         let logger = test_utils::TestLogger::new();
1614
1615         // Rebalance a bit so that we can send backwards from 3 to 2.
1616         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000);
1617
1618         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1619
1620         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1621         assert!(nodes[1].node.claim_funds(payment_preimage_1));
1622         check_added_monitors!(nodes[1], 1);
1623
1624         let (_, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
1625         {
1626                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1627                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1628                 nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1629                 check_added_monitors!(nodes[2], 1);
1630         }
1631
1632         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1633         // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1634         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1635
1636         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1637         assert_eq!(events.len(), 1);
1638         let payment_event = SendEvent::from_event(events.pop().unwrap());
1639         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1640         let events = nodes[1].node.get_and_clear_pending_msg_events();
1641         assert_eq!(events.len(), 0);
1642         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1643         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1644
1645         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1646         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1647         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1648
1649         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1650         assert_eq!(msg_events.len(), 1);
1651         match msg_events[0] {
1652                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1653                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1654                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1655                 },
1656                 _ => panic!("Unexpected event"),
1657         }
1658
1659         let events = nodes[2].node.get_and_clear_pending_events();
1660         assert_eq!(events.len(), 1);
1661         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1662                 assert_eq!(payment_hash, payment_hash_2);
1663                 assert!(!rejected_by_dest);
1664         } else { panic!("Unexpected event!"); }
1665
1666         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1667         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1668         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1669         check_added_monitors!(nodes[1], 0);
1670
1671         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1672         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1673         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1674
1675         let events = nodes[0].node.get_and_clear_pending_events();
1676         assert_eq!(events.len(), 1);
1677         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1678                 assert_eq!(payment_preimage, payment_preimage_1);
1679         } else { panic!("Unexpected event!"); }
1680 }
1681
1682 #[test]
1683 fn test_monitor_update_on_pending_forwards() {
1684         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1685         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1686         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1687         // from C to A will be pending a forward to A.
1688         let chanmon_cfgs = create_chanmon_cfgs(3);
1689         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1690         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1691         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1692         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1693         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1694         let logger = test_utils::TestLogger::new();
1695
1696         // Rebalance a bit so that we can send backwards from 3 to 1.
1697         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000);
1698
1699         let (_, payment_hash_1, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1700         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1701         expect_pending_htlcs_forwardable!(nodes[2]);
1702         check_added_monitors!(nodes[2], 1);
1703
1704         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1705         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1706         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1707         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1708
1709         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[0]);
1710         {
1711                 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1712                 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1713                 nodes[2].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1714                 check_added_monitors!(nodes[2], 1);
1715         }
1716
1717         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1718         assert_eq!(events.len(), 1);
1719         let payment_event = SendEvent::from_event(events.pop().unwrap());
1720         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1721         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1722
1723         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1724         expect_pending_htlcs_forwardable!(nodes[1]);
1725         check_added_monitors!(nodes[1], 1);
1726         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1727         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1728
1729         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1730         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1731         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1732         check_added_monitors!(nodes[1], 0);
1733
1734         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1735         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1736         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1737         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1738
1739         let events = nodes[0].node.get_and_clear_pending_events();
1740         assert_eq!(events.len(), 2);
1741         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1742                 assert_eq!(payment_hash, payment_hash_1);
1743                 assert!(rejected_by_dest);
1744         } else { panic!("Unexpected event!"); }
1745         match events[1] {
1746                 Event::PendingHTLCsForwardable { .. } => { },
1747                 _ => panic!("Unexpected event"),
1748         };
1749         nodes[0].node.process_pending_htlc_forwards();
1750         expect_payment_received!(nodes[0], payment_hash_2, payment_secret_2, 1000000);
1751
1752         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2);
1753 }
1754
1755 #[test]
1756 fn monitor_update_claim_fail_no_response() {
1757         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1758         // to channel being AwaitingRAA).
1759         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1760         // code was broken.
1761         let chanmon_cfgs = create_chanmon_cfgs(2);
1762         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1763         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1764         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1765         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1766         let logger = test_utils::TestLogger::new();
1767
1768         // Forward a payment for B to claim
1769         let (payment_preimage_1, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1770
1771         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1772         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(nodes[1]);
1773         {
1774                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1775                 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1776                 nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
1777                 check_added_monitors!(nodes[0], 1);
1778         }
1779
1780         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1781         assert_eq!(events.len(), 1);
1782         let payment_event = SendEvent::from_event(events.pop().unwrap());
1783         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1784         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1785
1786         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1787         assert!(nodes[1].node.claim_funds(payment_preimage_1));
1788         check_added_monitors!(nodes[1], 1);
1789         let events = nodes[1].node.get_and_clear_pending_msg_events();
1790         assert_eq!(events.len(), 0);
1791         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1792
1793         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1794         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1795         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1796         check_added_monitors!(nodes[1], 0);
1797         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1798
1799         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1800         check_added_monitors!(nodes[1], 1);
1801         expect_pending_htlcs_forwardable!(nodes[1]);
1802         expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 1000000);
1803
1804         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1805         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1806         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1807
1808         let events = nodes[0].node.get_and_clear_pending_events();
1809         assert_eq!(events.len(), 1);
1810         match events[0] {
1811                 Event::PaymentSent { ref payment_preimage } => {
1812                         assert_eq!(*payment_preimage, payment_preimage_1);
1813                 },
1814                 _ => panic!("Unexpected event"),
1815         }
1816
1817         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
1818 }
1819
1820 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1821 // restore_b_before_conf has no meaning if !confirm_a_first
1822 fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf: bool) {
1823         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1824         // the channel setup happily after the update is restored.
1825         let chanmon_cfgs = create_chanmon_cfgs(2);
1826         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1827         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1828         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1829
1830         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1831         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1832         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1833
1834         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1835
1836         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_tx.clone()).unwrap();
1837         check_added_monitors!(nodes[0], 0);
1838
1839         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1840         let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1841         let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1842         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1843         check_added_monitors!(nodes[1], 1);
1844
1845         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1846         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1847         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1848         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1849         check_added_monitors!(nodes[0], 1);
1850         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1851         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1852         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1853         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1854         check_added_monitors!(nodes[0], 0);
1855
1856         let events = nodes[0].node.get_and_clear_pending_events();
1857         assert_eq!(events.len(), 0);
1858         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
1859         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0)[0].txid(), funding_output.txid);
1860
1861         if confirm_a_first {
1862                 confirm_transaction(&nodes[0], &funding_tx);
1863                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1864         } else {
1865                 assert!(!restore_b_before_conf);
1866                 confirm_transaction(&nodes[1], &funding_tx);
1867                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1868         }
1869
1870         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1871         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1872         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1873         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1874         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1875         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1876
1877         if !restore_b_before_conf {
1878                 confirm_transaction(&nodes[1], &funding_tx);
1879                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1880                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1881         }
1882
1883         *nodes[1].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1884         let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1885         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1886         check_added_monitors!(nodes[1], 0);
1887
1888         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1889                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1890
1891                 confirm_transaction(&nodes[0], &funding_tx);
1892                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1893                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1894         } else {
1895                 if restore_b_before_conf {
1896                         confirm_transaction(&nodes[1], &funding_tx);
1897                 }
1898                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1899                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1900         };
1901         for node in nodes.iter() {
1902                 assert!(node.net_graph_msg_handler.handle_channel_announcement(&announcement).unwrap());
1903                 node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
1904                 node.net_graph_msg_handler.handle_channel_update(&bs_update).unwrap();
1905         }
1906
1907         send_payment(&nodes[0], &[&nodes[1]], 8000000);
1908         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1909 }
1910
1911 #[test]
1912 fn during_funding_monitor_fail() {
1913         do_during_funding_monitor_fail(true, true);
1914         do_during_funding_monitor_fail(true, false);
1915         do_during_funding_monitor_fail(false, false);
1916 }
1917
1918 #[test]
1919 fn test_path_paused_mpp() {
1920         // Simple test of sending a multi-part payment where one path is currently blocked awaiting
1921         // monitor update
1922         let chanmon_cfgs = create_chanmon_cfgs(4);
1923         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1924         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
1925         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1926
1927         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1928         let (chan_2_ann, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known());
1929         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1930         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1931         let logger = test_utils::TestLogger::new();
1932
1933         let (payment_preimage, payment_hash, payment_secret) = get_payment_preimage_hash!(&nodes[3]);
1934         let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), Some(InvoiceFeatures::known()), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
1935
1936         // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
1937         let path = route.paths[0].clone();
1938         route.paths.push(path);
1939         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
1940         route.paths[0][0].short_channel_id = chan_1_id;
1941         route.paths[0][1].short_channel_id = chan_3_id;
1942         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
1943         route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
1944         route.paths[1][1].short_channel_id = chan_4_id;
1945
1946         // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
1947         // (for the path 0 -> 2 -> 3) fails.
1948         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1949         *nodes[0].chain_monitor.next_update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1950
1951         // Now check that we get the right return value, indicating that the first path succeeded but
1952         // the second got a MonitorUpdateFailed err. This implies PaymentSendFailure::PartialFailure as
1953         // some paths succeeded, preventing retry.
1954         if let Err(PaymentSendFailure::PartialFailure(results)) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
1955                 assert_eq!(results.len(), 2);
1956                 if let Ok(()) = results[0] {} else { panic!(); }
1957                 if let Err(APIError::MonitorUpdateFailed) = results[1] {} else { panic!(); }
1958         } else { panic!(); }
1959         check_added_monitors!(nodes[0], 2);
1960         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Ok(()));
1961
1962         // Pass the first HTLC of the payment along to nodes[3].
1963         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1964         assert_eq!(events.len(), 1);
1965         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 0, payment_hash.clone(), payment_secret, events.pop().unwrap(), false);
1966
1967         // And check that, after we successfully update the monitor for chan_2 we can pass the second
1968         // HTLC along to nodes[3] and claim the whole payment back to nodes[0].
1969         let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2_id).unwrap().clone();
1970         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1971         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1972         assert_eq!(events.len(), 1);
1973         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash.clone(), payment_secret, events.pop().unwrap(), true);
1974
1975         claim_payment_along_route(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage);
1976 }
1977
1978 fn do_channel_holding_cell_serialize(disconnect: bool, reload_a: bool) {
1979         // Tests that, when we serialize a channel with AddHTLC entries in the holding cell, we
1980         // properly free them on reconnect. We previously failed such HTLCs upon serialization, but
1981         // that behavior was both somewhat unexpected and also broken (there was a debug assertion
1982         // which failed in such a case).
1983         let chanmon_cfgs = create_chanmon_cfgs(2);
1984         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1985         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1986         let persister: test_utils::TestPersister;
1987         let new_chain_monitor: test_utils::TestChainMonitor;
1988         let nodes_0_deserialized: ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
1989         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1990
1991         let chan_id = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 15_000_000, 7_000_000_000, InitFeatures::known(), InitFeatures::known()).2;
1992         let (payment_preimage_1, payment_hash_1, payment_secret_1) = get_payment_preimage_hash!(&nodes[1]);
1993         let (payment_preimage_2, payment_hash_2, payment_secret_2) = get_payment_preimage_hash!(&nodes[1]);
1994
1995         // Do a really complicated dance to get an HTLC into the holding cell, with MonitorUpdateFailed
1996         // set but AwaitingRemoteRevoke unset. When this test was written, any attempts to send an HTLC
1997         // while MonitorUpdateFailed is set are immediately failed-backwards. Thus, the only way to get
1998         // an AddHTLC into the holding cell is to add it while AwaitingRemoteRevoke is set but
1999         // MonitorUpdateFailed is unset, and then swap the flags.
2000         //
2001         // We do this by:
2002         //  a) routing a payment from node B to node A,
2003         //  b) sending a payment from node A to node B without delivering any of the generated messages,
2004         //     putting node A in AwaitingRemoteRevoke,
2005         //  c) sending a second payment from node A to node B, which is immediately placed in the
2006         //     holding cell,
2007         //  d) claiming the first payment from B, allowing us to fail the monitor update which occurs
2008         //     when we try to persist the payment preimage,
2009         //  e) delivering A's commitment_signed from (b) and the resulting B revoke_and_ack message,
2010         //     clearing AwaitingRemoteRevoke on node A.
2011         //
2012         // Note that because, at the end, MonitorUpdateFailed is still set, the HTLC generated in (c)
2013         // will not be freed from the holding cell.
2014         let (payment_preimage_0, _, _) = route_payment(&nodes[1], &[&nodes[0]], 100000);
2015
2016         let route = {
2017                 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
2018                 get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, None, &Vec::new(), 100000, TEST_FINAL_CLTV, nodes[0].logger).unwrap()
2019         };
2020
2021         nodes[0].node.send_payment(&route, payment_hash_1, &Some(payment_secret_1)).unwrap();
2022         check_added_monitors!(nodes[0], 1);
2023         let send = SendEvent::from_node(&nodes[0]);
2024         assert_eq!(send.msgs.len(), 1);
2025
2026         nodes[0].node.send_payment(&route, payment_hash_2, &Some(payment_secret_2)).unwrap();
2027         check_added_monitors!(nodes[0], 0);
2028
2029         *nodes[0].chain_monitor.update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
2030         assert!(nodes[0].node.claim_funds(payment_preimage_0));
2031         check_added_monitors!(nodes[0], 1);
2032
2033         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send.msgs[0]);
2034         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send.commitment_msg);
2035         check_added_monitors!(nodes[1], 1);
2036
2037         let (raa, cs) = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
2038
2039         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa);
2040         check_added_monitors!(nodes[0], 1);
2041
2042         if disconnect {
2043                 // Optionally reload nodes[0] entirely through a serialization roundtrip, otherwise just
2044                 // disconnect the peers. Note that the fuzzer originally found this issue because
2045                 // deserializing a ChannelManager in this state causes an assertion failure.
2046                 if reload_a {
2047                         let nodes_0_serialized = nodes[0].node.encode();
2048                         let mut chan_0_monitor_serialized = test_utils::TestVecWriter(Vec::new());
2049                         nodes[0].chain_monitor.chain_monitor.monitors.read().unwrap().iter().next().unwrap().1.write(&mut chan_0_monitor_serialized).unwrap();
2050
2051                         persister = test_utils::TestPersister::new();
2052                         let keys_manager = &chanmon_cfgs[0].keys_manager;
2053                         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster.clone(), nodes[0].logger, node_cfgs[0].fee_estimator, &persister, keys_manager);
2054                         nodes[0].chain_monitor = &new_chain_monitor;
2055                         let mut chan_0_monitor_read = &chan_0_monitor_serialized.0[..];
2056                         let (_, mut chan_0_monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(
2057                                 &mut chan_0_monitor_read, keys_manager).unwrap();
2058                         assert!(chan_0_monitor_read.is_empty());
2059
2060                         let mut nodes_0_read = &nodes_0_serialized[..];
2061                         let config = UserConfig::default();
2062                         nodes_0_deserialized = {
2063                                 let mut channel_monitors = HashMap::new();
2064                                 channel_monitors.insert(chan_0_monitor.get_funding_txo().0, &mut chan_0_monitor);
2065                                 <(BlockHash, ChannelManager<EnforcingSigner, &test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
2066                                         default_config: config,
2067                                         keys_manager,
2068                                         fee_estimator: node_cfgs[0].fee_estimator,
2069                                         chain_monitor: nodes[0].chain_monitor,
2070                                         tx_broadcaster: nodes[0].tx_broadcaster.clone(),
2071                                         logger: nodes[0].logger,
2072                                         channel_monitors,
2073                                 }).unwrap().1
2074                         };
2075                         nodes[0].node = &nodes_0_deserialized;
2076                         assert!(nodes_0_read.is_empty());
2077
2078                         nodes[0].chain_monitor.watch_channel(chan_0_monitor.get_funding_txo().0.clone(), chan_0_monitor).unwrap();
2079                         check_added_monitors!(nodes[0], 1);
2080                 } else {
2081                         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
2082                 }
2083                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
2084
2085                 // Now reconnect the two
2086                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
2087                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
2088                 assert_eq!(reestablish_1.len(), 1);
2089                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
2090                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
2091                 assert_eq!(reestablish_2.len(), 1);
2092
2093                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
2094                 let resp_1 = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
2095                 check_added_monitors!(nodes[1], 0);
2096
2097                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
2098                 let resp_0 = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
2099
2100                 assert!(resp_0.0.is_none());
2101                 assert!(resp_0.1.is_none());
2102                 assert!(resp_0.2.is_none());
2103                 assert!(resp_1.0.is_none());
2104                 assert!(resp_1.1.is_none());
2105
2106                 // Check that the freshly-generated cs is equal to the original (which we will deliver in a
2107                 // moment).
2108                 if let Some(pending_cs) = resp_1.2 {
2109                         assert!(pending_cs.update_add_htlcs.is_empty());
2110                         assert!(pending_cs.update_fail_htlcs.is_empty());
2111                         assert!(pending_cs.update_fulfill_htlcs.is_empty());
2112                         assert_eq!(pending_cs.commitment_signed, cs);
2113                 } else { panic!(); }
2114
2115                 // There should be no monitor updates as we are still pending awaiting a failed one.
2116                 check_added_monitors!(nodes[0], 0);
2117                 check_added_monitors!(nodes[1], 0);
2118         }
2119
2120         // If we finish updating the monitor, we should free the holding cell right away (this did
2121         // not occur prior to #756).
2122         *nodes[0].chain_monitor.update_ret.lock().unwrap() = None;
2123         let (funding_txo, mon_id) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_id).unwrap().clone();
2124         nodes[0].node.channel_monitor_updated(&funding_txo, mon_id);
2125
2126         // New outbound messages should be generated immediately upon a call to
2127         // get_and_clear_pending_msg_events (but not before).
2128         check_added_monitors!(nodes[0], 0);
2129         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
2130         check_added_monitors!(nodes[0], 1);
2131         assert_eq!(events.len(), 1);
2132
2133         // Deliver the pending in-flight CS
2134         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &cs);
2135         check_added_monitors!(nodes[0], 1);
2136
2137         let commitment_msg = match events.pop().unwrap() {
2138                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
2139                         assert_eq!(node_id, nodes[1].node.get_our_node_id());
2140                         assert!(updates.update_fail_htlcs.is_empty());
2141                         assert!(updates.update_fail_malformed_htlcs.is_empty());
2142                         assert!(updates.update_fee.is_none());
2143                         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
2144                         nodes[1].node.handle_update_fulfill_htlc(&nodes[0].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
2145                         expect_payment_sent!(nodes[1], payment_preimage_0);
2146                         assert_eq!(updates.update_add_htlcs.len(), 1);
2147                         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &updates.update_add_htlcs[0]);
2148                         updates.commitment_signed
2149                 },
2150                 _ => panic!("Unexpected event type!"),
2151         };
2152
2153         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &commitment_msg);
2154         check_added_monitors!(nodes[1], 1);
2155
2156         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
2157         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
2158         expect_pending_htlcs_forwardable!(nodes[1]);
2159         expect_payment_received!(nodes[1], payment_hash_1, payment_secret_1, 100000);
2160         check_added_monitors!(nodes[1], 1);
2161
2162         commitment_signed_dance!(nodes[1], nodes[0], (), false, true, false);
2163
2164         expect_pending_htlcs_forwardable!(nodes[1]);
2165         expect_payment_received!(nodes[1], payment_hash_2, payment_secret_2, 100000);
2166
2167         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
2168         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
2169 }
2170 #[test]
2171 fn channel_holding_cell_serialize() {
2172         do_channel_holding_cell_serialize(true, true);
2173         do_channel_holding_cell_serialize(true, false);
2174         do_channel_holding_cell_serialize(false, true); // last arg doesn't matter
2175 }