_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let node_cfgs = create_node_cfgs(2);
  23         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  24         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  25         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  26
  27         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  28         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  29
  30         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  31         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1, None) {} else { panic!(); }
  32         check_added_monitors!(nodes[0], 1);
  33
  34         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  35         assert_eq!(events_1.len(), 2);
  36         match events_1[0] {
  37                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  38                 _ => panic!("Unexpected event"),
  39         };
  40         match events_1[1] {
  41                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  42                 _ => panic!("Unexpected event"),
  43         };
  44
  45         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  46         // PaymentFailed event
  47
  48         assert_eq!(nodes[0].node.list_channels().len(), 0);
  49 }
  50
  51 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  52         // Test that we can recover from a simple temporary monitor update failure optionally with
  53         // a disconnect in between
  54         let node_cfgs = create_node_cfgs(2);
  55         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  56         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  57         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  58
  59         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  60         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  61
  62         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  63         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1, None) {} else { panic!(); }
  64         check_added_monitors!(nodes[0], 1);
  65
  66         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  67         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  68         assert_eq!(nodes[0].node.list_channels().len(), 1);
  69
  70         if disconnect {
  71                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  72                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  73                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  74         }
  75
  76         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  77         nodes[0].node.test_restore_channel_monitor();
  78         check_added_monitors!(nodes[0], 1);
  79
  80         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  81         assert_eq!(events_2.len(), 1);
  82         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  83         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  84         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  85         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  86
  87         expect_pending_htlcs_forwardable!(nodes[1]);
  88
  89         let events_3 = nodes[1].node.get_and_clear_pending_events();
  90         assert_eq!(events_3.len(), 1);
  91         match events_3[0] {
  92                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
  93                         assert_eq!(payment_hash_1, *payment_hash);
  94                         assert_eq!(*payment_secret, None);
  95                         assert_eq!(amt, 1000000);
  96                 },
  97                 _ => panic!("Unexpected event"),
  98         }
  99
 100         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 101
 102         // Now set it to failed again...
 103         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 104         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 105         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2, None) {} else { panic!(); }
 106         check_added_monitors!(nodes[0], 1);
 107
 108         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 109         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 110         assert_eq!(nodes[0].node.list_channels().len(), 1);
 111
 112         if disconnect {
 113                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 114                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 115                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 116         }
 117
 118         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 119         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 120         nodes[0].node.test_restore_channel_monitor();
 121         check_added_monitors!(nodes[0], 1);
 122         check_closed_broadcast!(nodes[0], false);
 123
 124         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 125         // PaymentFailed event
 126
 127         assert_eq!(nodes[0].node.list_channels().len(), 0);
 128 }
 129
 130 #[test]
 131 fn test_simple_monitor_temporary_update_fail() {
 132         do_test_simple_monitor_temporary_update_fail(false);
 133         do_test_simple_monitor_temporary_update_fail(true);
 134 }
 135
 136 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 137         let disconnect_flags = 8 | 16;
 138
 139         // Test that we can recover from a temporary monitor update failure with some in-flight
 140         // HTLCs going on at the same time potentially with some disconnection thrown in.
 141         // * First we route a payment, then get a temporary monitor update failure when trying to
 142         //   route a second payment. We then claim the first payment.
 143         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 144         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 145         //   the ChannelMonitor on a watchtower).
 146         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 147         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 148         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 149         //   disconnect_count & !disconnect_flags is 0).
 150         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 151         //   through message sending, potentially disconnect/reconnecting multiple times based on
 152         //   disconnect_count, to get the update_fulfill_htlc through.
 153         // * We then walk through more message exchanges to get the original update_add_htlc
 154         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 155         //   disconnect/reconnecting based on disconnect_count.
 156         let node_cfgs = create_node_cfgs(2);
 157         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 158         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 159         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 160
 161         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 162
 163         // Now try to send a second payment which will fail to send
 164         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 165         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 166
 167         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 168         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2, None) {} else { panic!(); }
 169         check_added_monitors!(nodes[0], 1);
 170
 171         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 172         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 173         assert_eq!(nodes[0].node.list_channels().len(), 1);
 174
 175         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 176         // but nodes[0] won't respond since it is frozen.
 177         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
 178         check_added_monitors!(nodes[1], 1);
 179         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 180         assert_eq!(events_2.len(), 1);
 181         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 182                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 183                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 184                         assert!(update_add_htlcs.is_empty());
 185                         assert_eq!(update_fulfill_htlcs.len(), 1);
 186                         assert!(update_fail_htlcs.is_empty());
 187                         assert!(update_fail_malformed_htlcs.is_empty());
 188                         assert!(update_fee.is_none());
 189
 190                         if (disconnect_count & 16) == 0 {
 191                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 192                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 193                                 assert_eq!(events_3.len(), 1);
 194                                 match events_3[0] {
 195                                         Event::PaymentSent { ref payment_preimage } => {
 196                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 197                                         },
 198                                         _ => panic!("Unexpected event"),
 199                                 }
 200
 201                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 202                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 203                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 204                         }
 205
 206                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 207                 },
 208                 _ => panic!("Unexpected event"),
 209         };
 210
 211         if disconnect_count & !disconnect_flags > 0 {
 212                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 213                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 214         }
 215
 216         // Now fix monitor updating...
 217         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 218         nodes[0].node.test_restore_channel_monitor();
 219         check_added_monitors!(nodes[0], 1);
 220
 221         macro_rules! disconnect_reconnect_peers { () => { {
 222                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 223                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 224
 225                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 226                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 227                 assert_eq!(reestablish_1.len(), 1);
 228                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 229                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 230                 assert_eq!(reestablish_2.len(), 1);
 231
 232                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 233                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 234                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 235                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 236
 237                 assert!(as_resp.0.is_none());
 238                 assert!(bs_resp.0.is_none());
 239
 240                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 241         } } }
 242
 243         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 244                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 245                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 246
 247                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 248                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 249                 assert_eq!(reestablish_1.len(), 1);
 250                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 251                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 252                 assert_eq!(reestablish_2.len(), 1);
 253
 254                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 255                 check_added_monitors!(nodes[0], 0);
 256                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 257                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 258                 check_added_monitors!(nodes[1], 0);
 259                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 260
 261                 assert!(as_resp.0.is_none());
 262                 assert!(bs_resp.0.is_none());
 263
 264                 assert!(bs_resp.1.is_none());
 265                 if (disconnect_count & 16) == 0 {
 266                         assert!(bs_resp.2.is_none());
 267
 268                         assert!(as_resp.1.is_some());
 269                         assert!(as_resp.2.is_some());
 270                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 271                 } else {
 272                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 273                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 274                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 275                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 276                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 277                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 278
 279                         assert!(as_resp.1.is_none());
 280
 281                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 282                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 283                         assert_eq!(events_3.len(), 1);
 284                         match events_3[0] {
 285                                 Event::PaymentSent { ref payment_preimage } => {
 286                                         assert_eq!(*payment_preimage, payment_preimage_1);
 287                                 },
 288                                 _ => panic!("Unexpected event"),
 289                         }
 290
 291                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 292                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 293                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 294                         check_added_monitors!(nodes[0], 1);
 295
 296                         as_resp.1 = Some(as_resp_raa);
 297                         bs_resp.2 = None;
 298                 }
 299
 300                 if disconnect_count & !disconnect_flags > 1 {
 301                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 302
 303                         if (disconnect_count & 16) == 0 {
 304                                 assert!(reestablish_1 == second_reestablish_1);
 305                                 assert!(reestablish_2 == second_reestablish_2);
 306                         }
 307                         assert!(as_resp == second_as_resp);
 308                         assert!(bs_resp == second_bs_resp);
 309                 }
 310
 311                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 312         } else {
 313                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 314                 assert_eq!(events_4.len(), 2);
 315                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 316                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 317                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 318                                 msg.clone()
 319                         },
 320                         _ => panic!("Unexpected event"),
 321                 })
 322         };
 323
 324         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 325
 326         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 327         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 328         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 329         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 330         check_added_monitors!(nodes[1], 1);
 331
 332         if disconnect_count & !disconnect_flags > 2 {
 333                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 334
 335                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 336                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 337
 338                 assert!(as_resp.2.is_none());
 339                 assert!(bs_resp.2.is_none());
 340         }
 341
 342         let as_commitment_update;
 343         let bs_second_commitment_update;
 344
 345         macro_rules! handle_bs_raa { () => {
 346                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 347                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 348                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 349                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 350                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 351                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 352                 assert!(as_commitment_update.update_fee.is_none());
 353                 check_added_monitors!(nodes[0], 1);
 354         } }
 355
 356         macro_rules! handle_initial_raa { () => {
 357                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 358                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 359                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 360                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 361                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 362                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 363                 assert!(bs_second_commitment_update.update_fee.is_none());
 364                 check_added_monitors!(nodes[1], 1);
 365         } }
 366
 367         if (disconnect_count & 8) == 0 {
 368                 handle_bs_raa!();
 369
 370                 if disconnect_count & !disconnect_flags > 3 {
 371                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 372
 373                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 374                         assert!(bs_resp.1.is_none());
 375
 376                         assert!(as_resp.2.unwrap() == as_commitment_update);
 377                         assert!(bs_resp.2.is_none());
 378
 379                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 380                 }
 381
 382                 handle_initial_raa!();
 383
 384                 if disconnect_count & !disconnect_flags > 4 {
 385                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 386
 387                         assert!(as_resp.1.is_none());
 388                         assert!(bs_resp.1.is_none());
 389
 390                         assert!(as_resp.2.unwrap() == as_commitment_update);
 391                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 392                 }
 393         } else {
 394                 handle_initial_raa!();
 395
 396                 if disconnect_count & !disconnect_flags > 3 {
 397                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 398
 399                         assert!(as_resp.1.is_none());
 400                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 401
 402                         assert!(as_resp.2.is_none());
 403                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 404
 405                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 406                 }
 407
 408                 handle_bs_raa!();
 409
 410                 if disconnect_count & !disconnect_flags > 4 {
 411                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 412
 413                         assert!(as_resp.1.is_none());
 414                         assert!(bs_resp.1.is_none());
 415
 416                         assert!(as_resp.2.unwrap() == as_commitment_update);
 417                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 418                 }
 419         }
 420
 421         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 422         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 423         // No commitment_signed so get_event_msg's assert(len == 1) passes
 424         check_added_monitors!(nodes[0], 1);
 425
 426         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 427         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 428         // No commitment_signed so get_event_msg's assert(len == 1) passes
 429         check_added_monitors!(nodes[1], 1);
 430
 431         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 432         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 433         check_added_monitors!(nodes[1], 1);
 434
 435         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 436         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 437         check_added_monitors!(nodes[0], 1);
 438
 439         expect_pending_htlcs_forwardable!(nodes[1]);
 440
 441         let events_5 = nodes[1].node.get_and_clear_pending_events();
 442         assert_eq!(events_5.len(), 1);
 443         match events_5[0] {
 444                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
 445                         assert_eq!(payment_hash_2, *payment_hash);
 446                         assert_eq!(*payment_secret, None);
 447                         assert_eq!(amt, 1000000);
 448                 },
 449                 _ => panic!("Unexpected event"),
 450         }
 451
 452         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 453 }
 454
 455 #[test]
 456 fn test_monitor_temporary_update_fail_a() {
 457         do_test_monitor_temporary_update_fail(0);
 458         do_test_monitor_temporary_update_fail(1);
 459         do_test_monitor_temporary_update_fail(2);
 460         do_test_monitor_temporary_update_fail(3);
 461         do_test_monitor_temporary_update_fail(4);
 462         do_test_monitor_temporary_update_fail(5);
 463 }
 464
 465 #[test]
 466 fn test_monitor_temporary_update_fail_b() {
 467         do_test_monitor_temporary_update_fail(2 | 8);
 468         do_test_monitor_temporary_update_fail(3 | 8);
 469         do_test_monitor_temporary_update_fail(4 | 8);
 470         do_test_monitor_temporary_update_fail(5 | 8);
 471 }
 472
 473 #[test]
 474 fn test_monitor_temporary_update_fail_c() {
 475         do_test_monitor_temporary_update_fail(1 | 16);
 476         do_test_monitor_temporary_update_fail(2 | 16);
 477         do_test_monitor_temporary_update_fail(3 | 16);
 478         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 479         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 480 }
 481
 482 #[test]
 483 fn test_monitor_update_fail_cs() {
 484         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 485         let node_cfgs = create_node_cfgs(2);
 486         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 487         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 488         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 489
 490         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 491         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 492         nodes[0].node.send_payment(route, our_payment_hash, None).unwrap();
 493         check_added_monitors!(nodes[0], 1);
 494
 495         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 496         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 497
 498         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 499         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 500         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 501         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 502         check_added_monitors!(nodes[1], 1);
 503         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 504
 505         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 506         nodes[1].node.test_restore_channel_monitor();
 507         check_added_monitors!(nodes[1], 1);
 508         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 509         assert_eq!(responses.len(), 2);
 510
 511         match responses[0] {
 512                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 513                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 514                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 515                         check_added_monitors!(nodes[0], 1);
 516                 },
 517                 _ => panic!("Unexpected event"),
 518         }
 519         match responses[1] {
 520                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 521                         assert!(updates.update_add_htlcs.is_empty());
 522                         assert!(updates.update_fulfill_htlcs.is_empty());
 523                         assert!(updates.update_fail_htlcs.is_empty());
 524                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 525                         assert!(updates.update_fee.is_none());
 526                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 527
 528                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 529                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 530                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 531                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 532                         check_added_monitors!(nodes[0], 1);
 533                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 534                 },
 535                 _ => panic!("Unexpected event"),
 536         }
 537
 538         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 539         nodes[0].node.test_restore_channel_monitor();
 540         check_added_monitors!(nodes[0], 1);
 541
 542         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 543         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 544         check_added_monitors!(nodes[1], 1);
 545
 546         expect_pending_htlcs_forwardable!(nodes[1]);
 547
 548         let events = nodes[1].node.get_and_clear_pending_events();
 549         assert_eq!(events.len(), 1);
 550         match events[0] {
 551                 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
 552                         assert_eq!(payment_hash, our_payment_hash);
 553                         assert_eq!(payment_secret, None);
 554                         assert_eq!(amt, 1000000);
 555                 },
 556                 _ => panic!("Unexpected event"),
 557         };
 558
 559         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 560 }
 561
 562 #[test]
 563 fn test_monitor_update_fail_no_rebroadcast() {
 564         // Tests handling of a monitor update failure when no message rebroadcasting on
 565         // test_restore_channel_monitor() is required. Backported from
 566         // chanmon_fail_consistency fuzz tests.
 567         let node_cfgs = create_node_cfgs(2);
 568         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 569         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 570         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 571
 572         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 573         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 574         nodes[0].node.send_payment(route, our_payment_hash, None).unwrap();
 575         check_added_monitors!(nodes[0], 1);
 576
 577         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 578         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 579         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 580
 581         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 582         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 583         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 584         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 585         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 586         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 587         check_added_monitors!(nodes[1], 1);
 588
 589         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 590         nodes[1].node.test_restore_channel_monitor();
 591         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 592         check_added_monitors!(nodes[1], 1);
 593         expect_pending_htlcs_forwardable!(nodes[1]);
 594
 595         let events = nodes[1].node.get_and_clear_pending_events();
 596         assert_eq!(events.len(), 1);
 597         match events[0] {
 598                 Event::PaymentReceived { payment_hash, .. } => {
 599                         assert_eq!(payment_hash, our_payment_hash);
 600                 },
 601                 _ => panic!("Unexpected event"),
 602         }
 603
 604         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 605 }
 606
 607 #[test]
 608 fn test_monitor_update_raa_while_paused() {
 609         // Tests handling of an RAA while monitor updating has already been marked failed.
 610         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 611         let node_cfgs = create_node_cfgs(2);
 612         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 613         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 614         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 615
 616         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 617
 618         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 619         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 620         nodes[0].node.send_payment(route, our_payment_hash_1, None).unwrap();
 621         check_added_monitors!(nodes[0], 1);
 622         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 623
 624         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 625         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 626         nodes[1].node.send_payment(route, our_payment_hash_2, None).unwrap();
 627         check_added_monitors!(nodes[1], 1);
 628         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 629
 630         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 631         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 632         check_added_monitors!(nodes[1], 1);
 633         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 634
 635         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 636         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 637         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 638         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 639         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 640         check_added_monitors!(nodes[0], 1);
 641
 642         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 643         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 644         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 645         check_added_monitors!(nodes[0], 1);
 646
 647         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 648         nodes[0].node.test_restore_channel_monitor();
 649         check_added_monitors!(nodes[0], 1);
 650
 651         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 652         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 653         check_added_monitors!(nodes[1], 1);
 654         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 655
 656         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 657         check_added_monitors!(nodes[1], 1);
 658         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 659
 660         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 661         check_added_monitors!(nodes[0], 1);
 662         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 663
 664         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 665         check_added_monitors!(nodes[0], 1);
 666         expect_pending_htlcs_forwardable!(nodes[0]);
 667         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 668
 669         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 670         check_added_monitors!(nodes[1], 1);
 671         expect_pending_htlcs_forwardable!(nodes[1]);
 672         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 673
 674         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 675         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 676 }
 677
 678 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 679         // Tests handling of a monitor update failure when processing an incoming RAA
 680         let node_cfgs = create_node_cfgs(3);
 681         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 682         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 683         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 684         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 685
 686         // Rebalance a bit so that we can send backwards from 2 to 1.
 687         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 688
 689         // Route a first payment that we'll fail backwards
 690         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 691
 692         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 693         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
 694         expect_pending_htlcs_forwardable!(nodes[2]);
 695         check_added_monitors!(nodes[2], 1);
 696
 697         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 698         assert!(updates.update_add_htlcs.is_empty());
 699         assert!(updates.update_fulfill_htlcs.is_empty());
 700         assert_eq!(updates.update_fail_htlcs.len(), 1);
 701         assert!(updates.update_fail_malformed_htlcs.is_empty());
 702         assert!(updates.update_fee.is_none());
 703         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 704
 705         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 706         check_added_monitors!(nodes[0], 0);
 707
 708         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 709         // holding cell.
 710         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 711         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 712         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
 713         check_added_monitors!(nodes[0], 1);
 714
 715         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 716         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 717         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 718
 719         expect_pending_htlcs_forwardable!(nodes[1]);
 720         check_added_monitors!(nodes[1], 0);
 721         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 722
 723         // Now fail monitor updating.
 724         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 725         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 726         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 727         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 728         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 729         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 730         check_added_monitors!(nodes[1], 1);
 731
 732         // Attempt to forward a third payment but fail due to the second channel being unavailable
 733         // for forwarding.
 734
 735         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 736         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 737         nodes[0].node.send_payment(route, payment_hash_3, None).unwrap();
 738         check_added_monitors!(nodes[0], 1);
 739
 740         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 741         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 742         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 743         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 744         check_added_monitors!(nodes[1], 0);
 745
 746         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 747         assert_eq!(events_2.len(), 1);
 748         match events_2.remove(0) {
 749                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 750                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 751                         assert!(updates.update_fulfill_htlcs.is_empty());
 752                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 753                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 754                         assert!(updates.update_add_htlcs.is_empty());
 755                         assert!(updates.update_fee.is_none());
 756
 757                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 758                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 759
 760                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 761                         assert_eq!(msg_events.len(), 1);
 762                         match msg_events[0] {
 763                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 764                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 765                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 766                                 },
 767                                 _ => panic!("Unexpected event"),
 768                         }
 769
 770                         let events = nodes[0].node.get_and_clear_pending_events();
 771                         assert_eq!(events.len(), 1);
 772                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 773                                 assert_eq!(payment_hash, payment_hash_3);
 774                                 assert!(!rejected_by_dest);
 775                         } else { panic!("Unexpected event!"); }
 776                 },
 777                 _ => panic!("Unexpected event type!"),
 778         };
 779
 780         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 781                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 782                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 783                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 784                 nodes[2].node.send_payment(route, payment_hash_4, None).unwrap();
 785                 check_added_monitors!(nodes[2], 1);
 786
 787                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 788                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 789                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 790                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 791                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 792                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 793                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 794                 (Some(payment_preimage_4), Some(payment_hash_4))
 795         } else { (None, None) };
 796
 797         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 798         // update_add update.
 799         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 800         nodes[1].node.test_restore_channel_monitor();
 801         check_added_monitors!(nodes[1], 1);
 802         expect_pending_htlcs_forwardable!(nodes[1]);
 803         check_added_monitors!(nodes[1], 1);
 804
 805         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 806         if test_ignore_second_cs {
 807                 assert_eq!(events_3.len(), 3);
 808         } else {
 809                 assert_eq!(events_3.len(), 2);
 810         }
 811
 812         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 813         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 814         let messages_a = match events_3.pop().unwrap() {
 815                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 816                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 817                         assert!(updates.update_fulfill_htlcs.is_empty());
 818                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 819                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 820                         assert!(updates.update_add_htlcs.is_empty());
 821                         assert!(updates.update_fee.is_none());
 822                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 823                 },
 824                 _ => panic!("Unexpected event type!"),
 825         };
 826         let raa = if test_ignore_second_cs {
 827                 match events_3.remove(1) {
 828                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 829                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 830                                 Some(msg.clone())
 831                         },
 832                         _ => panic!("Unexpected event"),
 833                 }
 834         } else { None };
 835         let send_event_b = SendEvent::from_event(events_3.remove(0));
 836         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 837
 838         // Now deliver the new messages...
 839
 840         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 841         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 842         let events_4 = nodes[0].node.get_and_clear_pending_events();
 843         assert_eq!(events_4.len(), 1);
 844         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 845                 assert_eq!(payment_hash, payment_hash_1);
 846                 assert!(rejected_by_dest);
 847         } else { panic!("Unexpected event!"); }
 848
 849         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 850         if test_ignore_second_cs {
 851                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 852                 check_added_monitors!(nodes[2], 1);
 853                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 854                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 855                 check_added_monitors!(nodes[2], 1);
 856                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 857                 assert!(bs_cs.update_add_htlcs.is_empty());
 858                 assert!(bs_cs.update_fail_htlcs.is_empty());
 859                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 860                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 861                 assert!(bs_cs.update_fee.is_none());
 862
 863                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 864                 check_added_monitors!(nodes[1], 1);
 865                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 866                 assert!(as_cs.update_add_htlcs.is_empty());
 867                 assert!(as_cs.update_fail_htlcs.is_empty());
 868                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 869                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 870                 assert!(as_cs.update_fee.is_none());
 871
 872                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 873                 check_added_monitors!(nodes[1], 1);
 874                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 875
 876                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 877                 check_added_monitors!(nodes[2], 1);
 878                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 879
 880                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 881                 check_added_monitors!(nodes[2], 1);
 882                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 883
 884                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 885                 check_added_monitors!(nodes[1], 1);
 886                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 887         } else {
 888                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 889         }
 890
 891         expect_pending_htlcs_forwardable!(nodes[2]);
 892
 893         let events_6 = nodes[2].node.get_and_clear_pending_events();
 894         assert_eq!(events_6.len(), 1);
 895         match events_6[0] {
 896                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 897                 _ => panic!("Unexpected event"),
 898         };
 899
 900         if test_ignore_second_cs {
 901                 expect_pending_htlcs_forwardable!(nodes[1]);
 902                 check_added_monitors!(nodes[1], 1);
 903
 904                 send_event = SendEvent::from_node(&nodes[1]);
 905                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 906                 assert_eq!(send_event.msgs.len(), 1);
 907                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 908                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 909
 910                 expect_pending_htlcs_forwardable!(nodes[0]);
 911
 912                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 913                 assert_eq!(events_9.len(), 1);
 914                 match events_9[0] {
 915                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 916                         _ => panic!("Unexpected event"),
 917                 };
 918                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 919         }
 920
 921         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 922 }
 923
 924 #[test]
 925 fn test_monitor_update_fail_raa() {
 926         do_test_monitor_update_fail_raa(false);
 927         do_test_monitor_update_fail_raa(true);
 928 }
 929
 930 #[test]
 931 fn test_monitor_update_fail_reestablish() {
 932         // Simple test for message retransmission after monitor update failure on
 933         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 934         // HTLCs).
 935         let node_cfgs = create_node_cfgs(3);
 936         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 937         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 938         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 939         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 940
 941         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 942
 943         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 944         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 945
 946         assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
 947         check_added_monitors!(nodes[2], 1);
 948         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 949         assert!(updates.update_add_htlcs.is_empty());
 950         assert!(updates.update_fail_htlcs.is_empty());
 951         assert!(updates.update_fail_malformed_htlcs.is_empty());
 952         assert!(updates.update_fee.is_none());
 953         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 954         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 955         check_added_monitors!(nodes[1], 1);
 956         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 957         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 958
 959         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 960         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 961         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 962
 963         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 964         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 965
 966         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 967
 968         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 969         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 970         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 971         check_added_monitors!(nodes[1], 1);
 972
 973         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 974         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 975
 976         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 977         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 978
 979         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 980         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 981
 982         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 983
 984         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 985         check_added_monitors!(nodes[1], 0);
 986         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 987
 988         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 989         nodes[1].node.test_restore_channel_monitor();
 990         check_added_monitors!(nodes[1], 1);
 991
 992         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 993         assert!(updates.update_add_htlcs.is_empty());
 994         assert!(updates.update_fail_htlcs.is_empty());
 995         assert!(updates.update_fail_malformed_htlcs.is_empty());
 996         assert!(updates.update_fee.is_none());
 997         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 998         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 999         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1000
1001         let events = nodes[0].node.get_and_clear_pending_events();
1002         assert_eq!(events.len(), 1);
1003         match events[0] {
1004                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1005                 _ => panic!("Unexpected event"),
1006         }
1007 }
1008
1009 #[test]
1010 fn raa_no_response_awaiting_raa_state() {
1011         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1012         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1013         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1014         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1015         let node_cfgs = create_node_cfgs(2);
1016         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1017         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1018         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1019
1020         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1021         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1022         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1023         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1024
1025         // Queue up two payments - one will be delivered right away, one immediately goes into the
1026         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1027         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1028         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1029         // generation during RAA while in monitor-update-failed state.
1030         nodes[0].node.send_payment(route.clone(), payment_hash_1, None).unwrap();
1031         check_added_monitors!(nodes[0], 1);
1032         nodes[0].node.send_payment(route.clone(), payment_hash_2, None).unwrap();
1033         check_added_monitors!(nodes[0], 0);
1034
1035         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1036         assert_eq!(events.len(), 1);
1037         let payment_event = SendEvent::from_event(events.pop().unwrap());
1038         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1039         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1040         check_added_monitors!(nodes[1], 1);
1041
1042         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1043         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1044         check_added_monitors!(nodes[0], 1);
1045         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1046         assert_eq!(events.len(), 1);
1047         let payment_event = SendEvent::from_event(events.pop().unwrap());
1048
1049         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1050         check_added_monitors!(nodes[0], 1);
1051         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1052
1053         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1054         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1055         // then restore channel monitor updates.
1056         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1057         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1058         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1059         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1060         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1061         check_added_monitors!(nodes[1], 1);
1062
1063         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1064         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1065         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1066         check_added_monitors!(nodes[1], 1);
1067
1068         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1069         nodes[1].node.test_restore_channel_monitor();
1070         // nodes[1] should be AwaitingRAA here!
1071         check_added_monitors!(nodes[1], 1);
1072         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1073         expect_pending_htlcs_forwardable!(nodes[1]);
1074         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1075
1076         // We send a third payment here, which is somewhat of a redundant test, but the
1077         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1078         // commitment transaction states) whereas here we can explicitly check for it.
1079         nodes[0].node.send_payment(route.clone(), payment_hash_3, None).unwrap();
1080         check_added_monitors!(nodes[0], 0);
1081         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1082
1083         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1084         check_added_monitors!(nodes[0], 1);
1085         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1086         assert_eq!(events.len(), 1);
1087         let payment_event = SendEvent::from_event(events.pop().unwrap());
1088
1089         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1090         check_added_monitors!(nodes[0], 1);
1091         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1092
1093         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1094         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1095         check_added_monitors!(nodes[1], 1);
1096         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1097
1098         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1099         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1100         check_added_monitors!(nodes[1], 1);
1101         expect_pending_htlcs_forwardable!(nodes[1]);
1102         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1103         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1104
1105         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1106         check_added_monitors!(nodes[0], 1);
1107
1108         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1109         check_added_monitors!(nodes[0], 1);
1110         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1111
1112         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1113         check_added_monitors!(nodes[1], 1);
1114         expect_pending_htlcs_forwardable!(nodes[1]);
1115         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1116
1117         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1118         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1119         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1120 }
1121
1122 #[test]
1123 fn claim_while_disconnected_monitor_update_fail() {
1124         // Test for claiming a payment while disconnected and then having the resulting
1125         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1126         // contrived case for nodes with network instability.
1127         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1128         // code introduced a regression in this test (specifically, this caught a removal of the
1129         // channel_reestablish handling ensuring the order was sensical given the messages used).
1130         let node_cfgs = create_node_cfgs(2);
1131         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1132         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1133         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1134
1135         // Forward a payment for B to claim
1136         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1137
1138         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1139         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1140
1141         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1142         check_added_monitors!(nodes[1], 1);
1143
1144         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1145         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1146
1147         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1148         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1149
1150         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1151         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1152
1153         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1154         // update.
1155         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1156
1157         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1158         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1159         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1160         check_added_monitors!(nodes[1], 1);
1161         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1162
1163         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1164         // the monitor still failed
1165         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1166         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1167         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1168         check_added_monitors!(nodes[0], 1);
1169
1170         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1171         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1172         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1173         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1174         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1175         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1176         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1177
1178         // Now un-fail the monitor, which will result in B sending its original commitment update,
1179         // receiving the commitment update from A, and the resulting commitment dances.
1180         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1181         nodes[1].node.test_restore_channel_monitor();
1182         check_added_monitors!(nodes[1], 1);
1183
1184         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1185         assert_eq!(bs_msgs.len(), 2);
1186
1187         match bs_msgs[0] {
1188                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1189                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1190                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1191                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1192                         check_added_monitors!(nodes[0], 1);
1193
1194                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1195                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1196                         check_added_monitors!(nodes[1], 1);
1197                 },
1198                 _ => panic!("Unexpected event"),
1199         }
1200
1201         match bs_msgs[1] {
1202                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1203                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1204                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1205                         check_added_monitors!(nodes[0], 1);
1206                 },
1207                 _ => panic!("Unexpected event"),
1208         }
1209
1210         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1211
1212         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1213         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1214         check_added_monitors!(nodes[0], 1);
1215         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1216
1217         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1218         check_added_monitors!(nodes[1], 1);
1219         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1220         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1221         check_added_monitors!(nodes[1], 1);
1222
1223         expect_pending_htlcs_forwardable!(nodes[1]);
1224         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1225
1226         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1227         check_added_monitors!(nodes[0], 1);
1228
1229         let events = nodes[0].node.get_and_clear_pending_events();
1230         assert_eq!(events.len(), 1);
1231         match events[0] {
1232                 Event::PaymentSent { ref payment_preimage } => {
1233                         assert_eq!(*payment_preimage, payment_preimage_1);
1234                 },
1235                 _ => panic!("Unexpected event"),
1236         }
1237
1238         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1239 }
1240
1241 #[test]
1242 fn monitor_failed_no_reestablish_response() {
1243         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1244         // response to a commitment_signed.
1245         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1246         // debug_assert!() failure in channel_reestablish handling.
1247         let node_cfgs = create_node_cfgs(2);
1248         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1249         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1250         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1251
1252         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1253         // on receipt).
1254         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1255         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1256         nodes[0].node.send_payment(route, payment_hash_1, None).unwrap();
1257         check_added_monitors!(nodes[0], 1);
1258
1259         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1260         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1261         assert_eq!(events.len(), 1);
1262         let payment_event = SendEvent::from_event(events.pop().unwrap());
1263         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1264         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1265         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1266         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1267         check_added_monitors!(nodes[1], 1);
1268
1269         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1270         // is still failing to update monitors.
1271         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1272         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1273
1274         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1275         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1276
1277         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1278         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1279
1280         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1281         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1282
1283         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1284         nodes[1].node.test_restore_channel_monitor();
1285         check_added_monitors!(nodes[1], 1);
1286         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1287
1288         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1289         check_added_monitors!(nodes[0], 1);
1290         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1291         check_added_monitors!(nodes[0], 1);
1292
1293         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1294         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1295         check_added_monitors!(nodes[1], 1);
1296
1297         expect_pending_htlcs_forwardable!(nodes[1]);
1298         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1299
1300         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1301 }
1302
1303 #[test]
1304 fn first_message_on_recv_ordering() {
1305         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1306         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1307         // a commitment_signed which needs to send an RAA first.
1308         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1309         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1310         // response. To do this, we start routing two payments, with the final RAA for the first being
1311         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1312         // have no pending response but will want to send a RAA/CS (with the updates for the second
1313         // payment applied).
1314         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1315         let node_cfgs = create_node_cfgs(2);
1316         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1317         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1318         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1319
1320         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1321         // can deliver it and fail the monitor update.
1322         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1323         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1324         nodes[0].node.send_payment(route, payment_hash_1, None).unwrap();
1325         check_added_monitors!(nodes[0], 1);
1326
1327         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1328         assert_eq!(events.len(), 1);
1329         let payment_event = SendEvent::from_event(events.pop().unwrap());
1330         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1331         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1332         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1333         check_added_monitors!(nodes[1], 1);
1334         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1335
1336         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1337         check_added_monitors!(nodes[0], 1);
1338         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1339         check_added_monitors!(nodes[0], 1);
1340
1341         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1342
1343         // Route the second payment, generating an update_add_htlc/commitment_signed
1344         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1345         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1346         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1347         check_added_monitors!(nodes[0], 1);
1348         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1349         assert_eq!(events.len(), 1);
1350         let payment_event = SendEvent::from_event(events.pop().unwrap());
1351         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1352
1353         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1354
1355         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1356         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1357         // to the next message also tests resetting the delivery order.
1358         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1359         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1360         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1361         check_added_monitors!(nodes[1], 1);
1362
1363         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1364         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1365         // the appropriate HTLC acceptance).
1366         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1367         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1368         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1369         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1370
1371         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1372         nodes[1].node.test_restore_channel_monitor();
1373         check_added_monitors!(nodes[1], 1);
1374
1375         expect_pending_htlcs_forwardable!(nodes[1]);
1376         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1377
1378         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1379         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1380         check_added_monitors!(nodes[0], 1);
1381         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1382         check_added_monitors!(nodes[0], 1);
1383
1384         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1385         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1386         check_added_monitors!(nodes[1], 1);
1387
1388         expect_pending_htlcs_forwardable!(nodes[1]);
1389         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1390
1391         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1392         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1393 }
1394
1395 #[test]
1396 fn test_monitor_update_fail_claim() {
1397         // Basic test for monitor update failures when processing claim_funds calls.
1398         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1399         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1400         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1401         // updating and claim the payment on B.
1402         let node_cfgs = create_node_cfgs(3);
1403         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1404         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1405         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1406         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1407
1408         // Rebalance a bit so that we can send backwards from 3 to 2.
1409         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1410
1411         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1412
1413         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1414         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1415         check_added_monitors!(nodes[1], 1);
1416
1417         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1418         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1419         nodes[2].node.send_payment(route, payment_hash_2, None).unwrap();
1420         check_added_monitors!(nodes[2], 1);
1421
1422         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1423         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1424         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1425
1426         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1427         assert_eq!(events.len(), 1);
1428         let payment_event = SendEvent::from_event(events.pop().unwrap());
1429         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1430         let events = nodes[1].node.get_and_clear_pending_msg_events();
1431         assert_eq!(events.len(), 0);
1432         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1433         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1434
1435         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1436         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1437         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1438
1439         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1440         assert_eq!(msg_events.len(), 1);
1441         match msg_events[0] {
1442                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1443                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1444                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1445                 },
1446                 _ => panic!("Unexpected event"),
1447         }
1448
1449         let events = nodes[2].node.get_and_clear_pending_events();
1450         assert_eq!(events.len(), 1);
1451         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1452                 assert_eq!(payment_hash, payment_hash_2);
1453                 assert!(!rejected_by_dest);
1454         } else { panic!("Unexpected event!"); }
1455
1456         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1457         nodes[1].node.test_restore_channel_monitor();
1458         check_added_monitors!(nodes[1], 1);
1459
1460         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1461         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1462         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1463
1464         let events = nodes[0].node.get_and_clear_pending_events();
1465         assert_eq!(events.len(), 1);
1466         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1467                 assert_eq!(payment_preimage, payment_preimage_1);
1468         } else { panic!("Unexpected event!"); }
1469 }
1470
1471 #[test]
1472 fn test_monitor_update_on_pending_forwards() {
1473         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1474         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1475         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1476         // from C to A will be pending a forward to A.
1477         let node_cfgs = create_node_cfgs(3);
1478         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1479         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1480         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1481         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1482
1483         // Rebalance a bit so that we can send backwards from 3 to 1.
1484         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1485
1486         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1487         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1488         expect_pending_htlcs_forwardable!(nodes[2]);
1489         check_added_monitors!(nodes[2], 1);
1490
1491         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1492         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1493         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1494         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1495
1496         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1497         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1498         nodes[2].node.send_payment(route, payment_hash_2, None).unwrap();
1499         check_added_monitors!(nodes[2], 1);
1500
1501         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1502         assert_eq!(events.len(), 1);
1503         let payment_event = SendEvent::from_event(events.pop().unwrap());
1504         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1505         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1506
1507         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1508         expect_pending_htlcs_forwardable!(nodes[1]);
1509         check_added_monitors!(nodes[1], 1);
1510         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1511         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1512
1513         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1514         nodes[1].node.test_restore_channel_monitor();
1515         check_added_monitors!(nodes[1], 1);
1516
1517         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1518         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1519         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1520         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1521
1522         let events = nodes[0].node.get_and_clear_pending_events();
1523         assert_eq!(events.len(), 2);
1524         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1525                 assert_eq!(payment_hash, payment_hash_1);
1526                 assert!(rejected_by_dest);
1527         } else { panic!("Unexpected event!"); }
1528         match events[1] {
1529                 Event::PendingHTLCsForwardable { .. } => { },
1530                 _ => panic!("Unexpected event"),
1531         };
1532         nodes[0].node.process_pending_htlc_forwards();
1533         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1534
1535         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1536 }
1537
1538 #[test]
1539 fn monitor_update_claim_fail_no_response() {
1540         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1541         // to channel being AwaitingRAA).
1542         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1543         // code was broken.
1544         let node_cfgs = create_node_cfgs(2);
1545         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1546         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1547         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1548
1549         // Forward a payment for B to claim
1550         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1551
1552         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1553         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1554         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1555         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1556         check_added_monitors!(nodes[0], 1);
1557
1558         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1559         assert_eq!(events.len(), 1);
1560         let payment_event = SendEvent::from_event(events.pop().unwrap());
1561         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1562         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1563
1564         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1565         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1566         check_added_monitors!(nodes[1], 1);
1567         let events = nodes[1].node.get_and_clear_pending_msg_events();
1568         assert_eq!(events.len(), 0);
1569         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1570
1571         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1572         nodes[1].node.test_restore_channel_monitor();
1573         check_added_monitors!(nodes[1], 1);
1574         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1575
1576         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1577         check_added_monitors!(nodes[1], 1);
1578         expect_pending_htlcs_forwardable!(nodes[1]);
1579         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1580
1581         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1582         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1583         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1584
1585         let events = nodes[0].node.get_and_clear_pending_events();
1586         assert_eq!(events.len(), 1);
1587         match events[0] {
1588                 Event::PaymentSent { ref payment_preimage } => {
1589                         assert_eq!(*payment_preimage, payment_preimage_1);
1590                 },
1591                 _ => panic!("Unexpected event"),
1592         }
1593
1594         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1595 }
1596
1597 // Note that restore_between_fails with !fail_on_generate is useless
1598 // Also note that !fail_on_generate && !fail_on_signed is useless
1599 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1600 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1601 // restore_b_before_conf has no meaning if !confirm_a_first
1602 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1603         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1604         // the channel setup happily after the update is restored.
1605         let node_cfgs = create_node_cfgs(2);
1606         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1607         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1608
1609         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1610         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1611         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1612
1613         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1614
1615         if fail_on_generate {
1616                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1617         }
1618         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1619         check_added_monitors!(nodes[0], 1);
1620
1621         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1622         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1623         check_added_monitors!(nodes[1], 1);
1624
1625         if restore_between_fails {
1626                 assert!(fail_on_generate);
1627                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1628                 nodes[0].node.test_restore_channel_monitor();
1629                 check_added_monitors!(nodes[0], 1);
1630                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1631                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1632         }
1633
1634         if fail_on_signed {
1635                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1636         } else {
1637                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1638                 assert!(fail_on_generate); // Somebody has to fail
1639         }
1640         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1641         if fail_on_signed || !restore_between_fails {
1642                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1643                 if fail_on_generate && !restore_between_fails {
1644                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1645                         check_added_monitors!(nodes[0], 0);
1646                 } else {
1647                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1648                         check_added_monitors!(nodes[0], 1);
1649                 }
1650                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1651                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1652                 nodes[0].node.test_restore_channel_monitor();
1653         }
1654
1655         check_added_monitors!(nodes[0], 1);
1656
1657         let events = nodes[0].node.get_and_clear_pending_events();
1658         assert_eq!(events.len(), 1);
1659         match events[0] {
1660                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1661                         assert_eq!(user_channel_id, 43);
1662                         assert_eq!(*funding_txo, funding_output);
1663                 },
1664                 _ => panic!("Unexpected event"),
1665         };
1666
1667         if confirm_a_first {
1668                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1669                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1670         } else {
1671                 assert!(!restore_b_before_conf);
1672                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1673                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1674         }
1675
1676         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1677         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1678         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1679         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1680         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1681         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1682
1683         if !restore_b_before_conf {
1684                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1685                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1686                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1687         }
1688
1689         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1690         nodes[1].node.test_restore_channel_monitor();
1691         check_added_monitors!(nodes[1], 1);
1692
1693         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1694                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1695
1696                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1697                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1698                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1699         } else {
1700                 if restore_b_before_conf {
1701                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1702                 }
1703                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1704                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1705         };
1706         for node in nodes.iter() {
1707                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1708                 node.router.handle_channel_update(&as_update).unwrap();
1709                 node.router.handle_channel_update(&bs_update).unwrap();
1710         }
1711
1712         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1713         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1714 }
1715
1716 #[test]
1717 fn during_funding_monitor_fail() {
1718         do_during_funding_monitor_fail(false, false, true, true, true);
1719         do_during_funding_monitor_fail(true, false, true, false, false);
1720         do_during_funding_monitor_fail(true, true, true, true, false);
1721         do_during_funding_monitor_fail(true, true, false, false, false);
1722 }