_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSendFailure};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let node_cfgs = create_node_cfgs(2);
  23         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  24         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  25         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  26
  27         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  28         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  29
  30         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  31         unwrap_send_err!(nodes[0].node.send_payment(route, payment_hash_1, None), true, APIError::ChannelUnavailable {..}, {});
  32         check_added_monitors!(nodes[0], 1);
  33
  34         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  35         assert_eq!(events_1.len(), 2);
  36         match events_1[0] {
  37                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  38                 _ => panic!("Unexpected event"),
  39         };
  40         match events_1[1] {
  41                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  42                 _ => panic!("Unexpected event"),
  43         };
  44
  45         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  46         // PaymentFailed event
  47
  48         assert_eq!(nodes[0].node.list_channels().len(), 0);
  49 }
  50
  51 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  52         // Test that we can recover from a simple temporary monitor update failure optionally with
  53         // a disconnect in between
  54         let node_cfgs = create_node_cfgs(2);
  55         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  56         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  57         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  58
  59         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  60         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  61
  62         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  63
  64         unwrap_send_err!(nodes[0].node.send_payment(route.clone(), payment_hash_1, None), false, APIError::MonitorUpdateFailed, {});
  65         check_added_monitors!(nodes[0], 1);
  66
  67         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  68         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  69         assert_eq!(nodes[0].node.list_channels().len(), 1);
  70
  71         if disconnect {
  72                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  73                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  74                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  75         }
  76
  77         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  78         nodes[0].node.test_restore_channel_monitor();
  79         check_added_monitors!(nodes[0], 1);
  80
  81         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  82         assert_eq!(events_2.len(), 1);
  83         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  84         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  85         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  86         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  87
  88         expect_pending_htlcs_forwardable!(nodes[1]);
  89
  90         let events_3 = nodes[1].node.get_and_clear_pending_events();
  91         assert_eq!(events_3.len(), 1);
  92         match events_3[0] {
  93                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
  94                         assert_eq!(payment_hash_1, *payment_hash);
  95                         assert_eq!(*payment_secret, None);
  96                         assert_eq!(amt, 1000000);
  97                 },
  98                 _ => panic!("Unexpected event"),
  99         }
 100
 101         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 102
 103         // Now set it to failed again...
 104         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 105         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 106         unwrap_send_err!(nodes[0].node.send_payment(route, payment_hash_2, None), false, APIError::MonitorUpdateFailed, {});
 107         check_added_monitors!(nodes[0], 1);
 108
 109         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 110         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 111         assert_eq!(nodes[0].node.list_channels().len(), 1);
 112
 113         if disconnect {
 114                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 115                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 116                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 117         }
 118
 119         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 120         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 121         nodes[0].node.test_restore_channel_monitor();
 122         check_added_monitors!(nodes[0], 1);
 123         check_closed_broadcast!(nodes[0], false);
 124
 125         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 126         // PaymentFailed event
 127
 128         assert_eq!(nodes[0].node.list_channels().len(), 0);
 129 }
 130
 131 #[test]
 132 fn test_simple_monitor_temporary_update_fail() {
 133         do_test_simple_monitor_temporary_update_fail(false);
 134         do_test_simple_monitor_temporary_update_fail(true);
 135 }
 136
 137 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 138         let disconnect_flags = 8 | 16;
 139
 140         // Test that we can recover from a temporary monitor update failure with some in-flight
 141         // HTLCs going on at the same time potentially with some disconnection thrown in.
 142         // * First we route a payment, then get a temporary monitor update failure when trying to
 143         //   route a second payment. We then claim the first payment.
 144         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 145         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 146         //   the ChannelMonitor on a watchtower).
 147         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 148         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 149         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 150         //   disconnect_count & !disconnect_flags is 0).
 151         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 152         //   through message sending, potentially disconnect/reconnecting multiple times based on
 153         //   disconnect_count, to get the update_fulfill_htlc through.
 154         // * We then walk through more message exchanges to get the original update_add_htlc
 155         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 156         //   disconnect/reconnecting based on disconnect_count.
 157         let node_cfgs = create_node_cfgs(2);
 158         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 159         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 160         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 161
 162         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 163
 164         // Now try to send a second payment which will fail to send
 165         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 166         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 167
 168         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 169         unwrap_send_err!(nodes[0].node.send_payment(route.clone(), payment_hash_2, None), false, APIError::MonitorUpdateFailed, {});
 170         check_added_monitors!(nodes[0], 1);
 171
 172         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 173         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 174         assert_eq!(nodes[0].node.list_channels().len(), 1);
 175
 176         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 177         // but nodes[0] won't respond since it is frozen.
 178         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
 179         check_added_monitors!(nodes[1], 1);
 180         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 181         assert_eq!(events_2.len(), 1);
 182         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 183                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 184                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 185                         assert!(update_add_htlcs.is_empty());
 186                         assert_eq!(update_fulfill_htlcs.len(), 1);
 187                         assert!(update_fail_htlcs.is_empty());
 188                         assert!(update_fail_malformed_htlcs.is_empty());
 189                         assert!(update_fee.is_none());
 190
 191                         if (disconnect_count & 16) == 0 {
 192                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 193                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 194                                 assert_eq!(events_3.len(), 1);
 195                                 match events_3[0] {
 196                                         Event::PaymentSent { ref payment_preimage } => {
 197                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 198                                         },
 199                                         _ => panic!("Unexpected event"),
 200                                 }
 201
 202                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 203                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 204                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 205                         }
 206
 207                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 208                 },
 209                 _ => panic!("Unexpected event"),
 210         };
 211
 212         if disconnect_count & !disconnect_flags > 0 {
 213                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 214                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 215         }
 216
 217         // Now fix monitor updating...
 218         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 219         nodes[0].node.test_restore_channel_monitor();
 220         check_added_monitors!(nodes[0], 1);
 221
 222         macro_rules! disconnect_reconnect_peers { () => { {
 223                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 224                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 225
 226                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 227                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 228                 assert_eq!(reestablish_1.len(), 1);
 229                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 230                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 231                 assert_eq!(reestablish_2.len(), 1);
 232
 233                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 234                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 235                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 236                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 237
 238                 assert!(as_resp.0.is_none());
 239                 assert!(bs_resp.0.is_none());
 240
 241                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 242         } } }
 243
 244         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 245                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 246                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 247
 248                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 249                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 250                 assert_eq!(reestablish_1.len(), 1);
 251                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 252                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 253                 assert_eq!(reestablish_2.len(), 1);
 254
 255                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 256                 check_added_monitors!(nodes[0], 0);
 257                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 258                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 259                 check_added_monitors!(nodes[1], 0);
 260                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 261
 262                 assert!(as_resp.0.is_none());
 263                 assert!(bs_resp.0.is_none());
 264
 265                 assert!(bs_resp.1.is_none());
 266                 if (disconnect_count & 16) == 0 {
 267                         assert!(bs_resp.2.is_none());
 268
 269                         assert!(as_resp.1.is_some());
 270                         assert!(as_resp.2.is_some());
 271                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 272                 } else {
 273                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 274                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 275                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 276                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 277                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 278                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 279
 280                         assert!(as_resp.1.is_none());
 281
 282                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 283                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 284                         assert_eq!(events_3.len(), 1);
 285                         match events_3[0] {
 286                                 Event::PaymentSent { ref payment_preimage } => {
 287                                         assert_eq!(*payment_preimage, payment_preimage_1);
 288                                 },
 289                                 _ => panic!("Unexpected event"),
 290                         }
 291
 292                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 293                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 294                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 295                         check_added_monitors!(nodes[0], 1);
 296
 297                         as_resp.1 = Some(as_resp_raa);
 298                         bs_resp.2 = None;
 299                 }
 300
 301                 if disconnect_count & !disconnect_flags > 1 {
 302                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 303
 304                         if (disconnect_count & 16) == 0 {
 305                                 assert!(reestablish_1 == second_reestablish_1);
 306                                 assert!(reestablish_2 == second_reestablish_2);
 307                         }
 308                         assert!(as_resp == second_as_resp);
 309                         assert!(bs_resp == second_bs_resp);
 310                 }
 311
 312                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 313         } else {
 314                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 315                 assert_eq!(events_4.len(), 2);
 316                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 317                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 318                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 319                                 msg.clone()
 320                         },
 321                         _ => panic!("Unexpected event"),
 322                 })
 323         };
 324
 325         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 326
 327         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 328         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 329         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 330         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 331         check_added_monitors!(nodes[1], 1);
 332
 333         if disconnect_count & !disconnect_flags > 2 {
 334                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 335
 336                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 337                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 338
 339                 assert!(as_resp.2.is_none());
 340                 assert!(bs_resp.2.is_none());
 341         }
 342
 343         let as_commitment_update;
 344         let bs_second_commitment_update;
 345
 346         macro_rules! handle_bs_raa { () => {
 347                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 348                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 349                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 350                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 351                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 352                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 353                 assert!(as_commitment_update.update_fee.is_none());
 354                 check_added_monitors!(nodes[0], 1);
 355         } }
 356
 357         macro_rules! handle_initial_raa { () => {
 358                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 359                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 360                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 361                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 362                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 363                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 364                 assert!(bs_second_commitment_update.update_fee.is_none());
 365                 check_added_monitors!(nodes[1], 1);
 366         } }
 367
 368         if (disconnect_count & 8) == 0 {
 369                 handle_bs_raa!();
 370
 371                 if disconnect_count & !disconnect_flags > 3 {
 372                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 373
 374                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 375                         assert!(bs_resp.1.is_none());
 376
 377                         assert!(as_resp.2.unwrap() == as_commitment_update);
 378                         assert!(bs_resp.2.is_none());
 379
 380                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 381                 }
 382
 383                 handle_initial_raa!();
 384
 385                 if disconnect_count & !disconnect_flags > 4 {
 386                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 387
 388                         assert!(as_resp.1.is_none());
 389                         assert!(bs_resp.1.is_none());
 390
 391                         assert!(as_resp.2.unwrap() == as_commitment_update);
 392                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 393                 }
 394         } else {
 395                 handle_initial_raa!();
 396
 397                 if disconnect_count & !disconnect_flags > 3 {
 398                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 399
 400                         assert!(as_resp.1.is_none());
 401                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 402
 403                         assert!(as_resp.2.is_none());
 404                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 405
 406                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 407                 }
 408
 409                 handle_bs_raa!();
 410
 411                 if disconnect_count & !disconnect_flags > 4 {
 412                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 413
 414                         assert!(as_resp.1.is_none());
 415                         assert!(bs_resp.1.is_none());
 416
 417                         assert!(as_resp.2.unwrap() == as_commitment_update);
 418                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 419                 }
 420         }
 421
 422         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 423         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 424         // No commitment_signed so get_event_msg's assert(len == 1) passes
 425         check_added_monitors!(nodes[0], 1);
 426
 427         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 428         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 429         // No commitment_signed so get_event_msg's assert(len == 1) passes
 430         check_added_monitors!(nodes[1], 1);
 431
 432         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 433         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 434         check_added_monitors!(nodes[1], 1);
 435
 436         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 437         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 438         check_added_monitors!(nodes[0], 1);
 439
 440         expect_pending_htlcs_forwardable!(nodes[1]);
 441
 442         let events_5 = nodes[1].node.get_and_clear_pending_events();
 443         assert_eq!(events_5.len(), 1);
 444         match events_5[0] {
 445                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
 446                         assert_eq!(payment_hash_2, *payment_hash);
 447                         assert_eq!(*payment_secret, None);
 448                         assert_eq!(amt, 1000000);
 449                 },
 450                 _ => panic!("Unexpected event"),
 451         }
 452
 453         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 454 }
 455
 456 #[test]
 457 fn test_monitor_temporary_update_fail_a() {
 458         do_test_monitor_temporary_update_fail(0);
 459         do_test_monitor_temporary_update_fail(1);
 460         do_test_monitor_temporary_update_fail(2);
 461         do_test_monitor_temporary_update_fail(3);
 462         do_test_monitor_temporary_update_fail(4);
 463         do_test_monitor_temporary_update_fail(5);
 464 }
 465
 466 #[test]
 467 fn test_monitor_temporary_update_fail_b() {
 468         do_test_monitor_temporary_update_fail(2 | 8);
 469         do_test_monitor_temporary_update_fail(3 | 8);
 470         do_test_monitor_temporary_update_fail(4 | 8);
 471         do_test_monitor_temporary_update_fail(5 | 8);
 472 }
 473
 474 #[test]
 475 fn test_monitor_temporary_update_fail_c() {
 476         do_test_monitor_temporary_update_fail(1 | 16);
 477         do_test_monitor_temporary_update_fail(2 | 16);
 478         do_test_monitor_temporary_update_fail(3 | 16);
 479         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 480         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 481 }
 482
 483 #[test]
 484 fn test_monitor_update_fail_cs() {
 485         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 486         let node_cfgs = create_node_cfgs(2);
 487         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 488         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 489         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 490
 491         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 492         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 493         nodes[0].node.send_payment(route, our_payment_hash, None).unwrap();
 494         check_added_monitors!(nodes[0], 1);
 495
 496         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 497         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 498
 499         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 500         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 501         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 502         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 503         check_added_monitors!(nodes[1], 1);
 504         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 505
 506         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 507         nodes[1].node.test_restore_channel_monitor();
 508         check_added_monitors!(nodes[1], 1);
 509         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 510         assert_eq!(responses.len(), 2);
 511
 512         match responses[0] {
 513                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 514                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 515                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 516                         check_added_monitors!(nodes[0], 1);
 517                 },
 518                 _ => panic!("Unexpected event"),
 519         }
 520         match responses[1] {
 521                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 522                         assert!(updates.update_add_htlcs.is_empty());
 523                         assert!(updates.update_fulfill_htlcs.is_empty());
 524                         assert!(updates.update_fail_htlcs.is_empty());
 525                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 526                         assert!(updates.update_fee.is_none());
 527                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 528
 529                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 530                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 531                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 532                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 533                         check_added_monitors!(nodes[0], 1);
 534                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 535                 },
 536                 _ => panic!("Unexpected event"),
 537         }
 538
 539         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 540         nodes[0].node.test_restore_channel_monitor();
 541         check_added_monitors!(nodes[0], 1);
 542
 543         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 544         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 545         check_added_monitors!(nodes[1], 1);
 546
 547         expect_pending_htlcs_forwardable!(nodes[1]);
 548
 549         let events = nodes[1].node.get_and_clear_pending_events();
 550         assert_eq!(events.len(), 1);
 551         match events[0] {
 552                 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
 553                         assert_eq!(payment_hash, our_payment_hash);
 554                         assert_eq!(payment_secret, None);
 555                         assert_eq!(amt, 1000000);
 556                 },
 557                 _ => panic!("Unexpected event"),
 558         };
 559
 560         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 561 }
 562
 563 #[test]
 564 fn test_monitor_update_fail_no_rebroadcast() {
 565         // Tests handling of a monitor update failure when no message rebroadcasting on
 566         // test_restore_channel_monitor() is required. Backported from
 567         // chanmon_fail_consistency fuzz tests.
 568         let node_cfgs = create_node_cfgs(2);
 569         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 570         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 571         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 572
 573         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 574         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 575         nodes[0].node.send_payment(route, our_payment_hash, None).unwrap();
 576         check_added_monitors!(nodes[0], 1);
 577
 578         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 579         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 580         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 581
 582         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 583         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 584         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 585         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 586         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 587         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 588         check_added_monitors!(nodes[1], 1);
 589
 590         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 591         nodes[1].node.test_restore_channel_monitor();
 592         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 593         check_added_monitors!(nodes[1], 1);
 594         expect_pending_htlcs_forwardable!(nodes[1]);
 595
 596         let events = nodes[1].node.get_and_clear_pending_events();
 597         assert_eq!(events.len(), 1);
 598         match events[0] {
 599                 Event::PaymentReceived { payment_hash, .. } => {
 600                         assert_eq!(payment_hash, our_payment_hash);
 601                 },
 602                 _ => panic!("Unexpected event"),
 603         }
 604
 605         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 606 }
 607
 608 #[test]
 609 fn test_monitor_update_raa_while_paused() {
 610         // Tests handling of an RAA while monitor updating has already been marked failed.
 611         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 612         let node_cfgs = create_node_cfgs(2);
 613         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 614         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 615         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 616
 617         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 618
 619         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 620         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 621         nodes[0].node.send_payment(route, our_payment_hash_1, None).unwrap();
 622         check_added_monitors!(nodes[0], 1);
 623         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 624
 625         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 626         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 627         nodes[1].node.send_payment(route, our_payment_hash_2, None).unwrap();
 628         check_added_monitors!(nodes[1], 1);
 629         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 630
 631         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 632         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 633         check_added_monitors!(nodes[1], 1);
 634         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 635
 636         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 637         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 638         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 639         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 640         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 641         check_added_monitors!(nodes[0], 1);
 642
 643         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 644         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 645         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 646         check_added_monitors!(nodes[0], 1);
 647
 648         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 649         nodes[0].node.test_restore_channel_monitor();
 650         check_added_monitors!(nodes[0], 1);
 651
 652         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 653         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 654         check_added_monitors!(nodes[1], 1);
 655         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 656
 657         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 658         check_added_monitors!(nodes[1], 1);
 659         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 660
 661         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 662         check_added_monitors!(nodes[0], 1);
 663         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 664
 665         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 666         check_added_monitors!(nodes[0], 1);
 667         expect_pending_htlcs_forwardable!(nodes[0]);
 668         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 669
 670         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 671         check_added_monitors!(nodes[1], 1);
 672         expect_pending_htlcs_forwardable!(nodes[1]);
 673         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 674
 675         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 676         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 677 }
 678
 679 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 680         // Tests handling of a monitor update failure when processing an incoming RAA
 681         let node_cfgs = create_node_cfgs(3);
 682         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 683         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 684         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 685         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 686
 687         // Rebalance a bit so that we can send backwards from 2 to 1.
 688         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 689
 690         // Route a first payment that we'll fail backwards
 691         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 692
 693         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 694         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
 695         expect_pending_htlcs_forwardable!(nodes[2]);
 696         check_added_monitors!(nodes[2], 1);
 697
 698         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 699         assert!(updates.update_add_htlcs.is_empty());
 700         assert!(updates.update_fulfill_htlcs.is_empty());
 701         assert_eq!(updates.update_fail_htlcs.len(), 1);
 702         assert!(updates.update_fail_malformed_htlcs.is_empty());
 703         assert!(updates.update_fee.is_none());
 704         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 705
 706         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 707         check_added_monitors!(nodes[0], 0);
 708
 709         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 710         // holding cell.
 711         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 712         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 713         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
 714         check_added_monitors!(nodes[0], 1);
 715
 716         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 717         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 718         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 719
 720         expect_pending_htlcs_forwardable!(nodes[1]);
 721         check_added_monitors!(nodes[1], 0);
 722         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 723
 724         // Now fail monitor updating.
 725         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 726         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 727         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 728         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 729         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 730         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 731         check_added_monitors!(nodes[1], 1);
 732
 733         // Attempt to forward a third payment but fail due to the second channel being unavailable
 734         // for forwarding.
 735
 736         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 737         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 738         nodes[0].node.send_payment(route, payment_hash_3, None).unwrap();
 739         check_added_monitors!(nodes[0], 1);
 740
 741         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 742         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 743         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 744         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 745         check_added_monitors!(nodes[1], 0);
 746
 747         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 748         assert_eq!(events_2.len(), 1);
 749         match events_2.remove(0) {
 750                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 751                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 752                         assert!(updates.update_fulfill_htlcs.is_empty());
 753                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 754                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 755                         assert!(updates.update_add_htlcs.is_empty());
 756                         assert!(updates.update_fee.is_none());
 757
 758                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 759                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 760
 761                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 762                         assert_eq!(msg_events.len(), 1);
 763                         match msg_events[0] {
 764                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 765                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 766                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 767                                 },
 768                                 _ => panic!("Unexpected event"),
 769                         }
 770
 771                         let events = nodes[0].node.get_and_clear_pending_events();
 772                         assert_eq!(events.len(), 1);
 773                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 774                                 assert_eq!(payment_hash, payment_hash_3);
 775                                 assert!(!rejected_by_dest);
 776                         } else { panic!("Unexpected event!"); }
 777                 },
 778                 _ => panic!("Unexpected event type!"),
 779         };
 780
 781         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 782                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 783                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 784                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 785                 nodes[2].node.send_payment(route, payment_hash_4, None).unwrap();
 786                 check_added_monitors!(nodes[2], 1);
 787
 788                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 789                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 790                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 791                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 792                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 793                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 794                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 795                 (Some(payment_preimage_4), Some(payment_hash_4))
 796         } else { (None, None) };
 797
 798         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 799         // update_add update.
 800         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 801         nodes[1].node.test_restore_channel_monitor();
 802         check_added_monitors!(nodes[1], 1);
 803         expect_pending_htlcs_forwardable!(nodes[1]);
 804         check_added_monitors!(nodes[1], 1);
 805
 806         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 807         if test_ignore_second_cs {
 808                 assert_eq!(events_3.len(), 3);
 809         } else {
 810                 assert_eq!(events_3.len(), 2);
 811         }
 812
 813         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 814         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 815         let messages_a = match events_3.pop().unwrap() {
 816                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 817                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 818                         assert!(updates.update_fulfill_htlcs.is_empty());
 819                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 820                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 821                         assert!(updates.update_add_htlcs.is_empty());
 822                         assert!(updates.update_fee.is_none());
 823                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 824                 },
 825                 _ => panic!("Unexpected event type!"),
 826         };
 827         let raa = if test_ignore_second_cs {
 828                 match events_3.remove(1) {
 829                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 830                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 831                                 Some(msg.clone())
 832                         },
 833                         _ => panic!("Unexpected event"),
 834                 }
 835         } else { None };
 836         let send_event_b = SendEvent::from_event(events_3.remove(0));
 837         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 838
 839         // Now deliver the new messages...
 840
 841         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 842         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 843         let events_4 = nodes[0].node.get_and_clear_pending_events();
 844         assert_eq!(events_4.len(), 1);
 845         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 846                 assert_eq!(payment_hash, payment_hash_1);
 847                 assert!(rejected_by_dest);
 848         } else { panic!("Unexpected event!"); }
 849
 850         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 851         if test_ignore_second_cs {
 852                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 853                 check_added_monitors!(nodes[2], 1);
 854                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 855                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 856                 check_added_monitors!(nodes[2], 1);
 857                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 858                 assert!(bs_cs.update_add_htlcs.is_empty());
 859                 assert!(bs_cs.update_fail_htlcs.is_empty());
 860                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 861                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 862                 assert!(bs_cs.update_fee.is_none());
 863
 864                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 865                 check_added_monitors!(nodes[1], 1);
 866                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 867                 assert!(as_cs.update_add_htlcs.is_empty());
 868                 assert!(as_cs.update_fail_htlcs.is_empty());
 869                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 870                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 871                 assert!(as_cs.update_fee.is_none());
 872
 873                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 874                 check_added_monitors!(nodes[1], 1);
 875                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 876
 877                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 878                 check_added_monitors!(nodes[2], 1);
 879                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 880
 881                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 882                 check_added_monitors!(nodes[2], 1);
 883                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 884
 885                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 886                 check_added_monitors!(nodes[1], 1);
 887                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 888         } else {
 889                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 890         }
 891
 892         expect_pending_htlcs_forwardable!(nodes[2]);
 893
 894         let events_6 = nodes[2].node.get_and_clear_pending_events();
 895         assert_eq!(events_6.len(), 1);
 896         match events_6[0] {
 897                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 898                 _ => panic!("Unexpected event"),
 899         };
 900
 901         if test_ignore_second_cs {
 902                 expect_pending_htlcs_forwardable!(nodes[1]);
 903                 check_added_monitors!(nodes[1], 1);
 904
 905                 send_event = SendEvent::from_node(&nodes[1]);
 906                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 907                 assert_eq!(send_event.msgs.len(), 1);
 908                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 909                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 910
 911                 expect_pending_htlcs_forwardable!(nodes[0]);
 912
 913                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 914                 assert_eq!(events_9.len(), 1);
 915                 match events_9[0] {
 916                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 917                         _ => panic!("Unexpected event"),
 918                 };
 919                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 920         }
 921
 922         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 923 }
 924
 925 #[test]
 926 fn test_monitor_update_fail_raa() {
 927         do_test_monitor_update_fail_raa(false);
 928         do_test_monitor_update_fail_raa(true);
 929 }
 930
 931 #[test]
 932 fn test_monitor_update_fail_reestablish() {
 933         // Simple test for message retransmission after monitor update failure on
 934         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 935         // HTLCs).
 936         let node_cfgs = create_node_cfgs(3);
 937         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 938         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 939         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 940         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 941
 942         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 943
 944         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 945         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 946
 947         assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
 948         check_added_monitors!(nodes[2], 1);
 949         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 950         assert!(updates.update_add_htlcs.is_empty());
 951         assert!(updates.update_fail_htlcs.is_empty());
 952         assert!(updates.update_fail_malformed_htlcs.is_empty());
 953         assert!(updates.update_fee.is_none());
 954         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 955         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 956         check_added_monitors!(nodes[1], 1);
 957         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 958         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 959
 960         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 961         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 962         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 963
 964         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 965         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 966
 967         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 968
 969         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 970         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 971         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 972         check_added_monitors!(nodes[1], 1);
 973
 974         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 975         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 976
 977         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 978         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 979
 980         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 981         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 982
 983         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 984
 985         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 986         check_added_monitors!(nodes[1], 0);
 987         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 988
 989         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 990         nodes[1].node.test_restore_channel_monitor();
 991         check_added_monitors!(nodes[1], 1);
 992
 993         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 994         assert!(updates.update_add_htlcs.is_empty());
 995         assert!(updates.update_fail_htlcs.is_empty());
 996         assert!(updates.update_fail_malformed_htlcs.is_empty());
 997         assert!(updates.update_fee.is_none());
 998         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 999         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1000         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1001
1002         let events = nodes[0].node.get_and_clear_pending_events();
1003         assert_eq!(events.len(), 1);
1004         match events[0] {
1005                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1006                 _ => panic!("Unexpected event"),
1007         }
1008 }
1009
1010 #[test]
1011 fn raa_no_response_awaiting_raa_state() {
1012         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1013         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1014         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1015         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1016         let node_cfgs = create_node_cfgs(2);
1017         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1018         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1019         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1020
1021         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1022         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1023         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1024         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1025
1026         // Queue up two payments - one will be delivered right away, one immediately goes into the
1027         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1028         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1029         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1030         // generation during RAA while in monitor-update-failed state.
1031         nodes[0].node.send_payment(route.clone(), payment_hash_1, None).unwrap();
1032         check_added_monitors!(nodes[0], 1);
1033         nodes[0].node.send_payment(route.clone(), payment_hash_2, None).unwrap();
1034         check_added_monitors!(nodes[0], 0);
1035
1036         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1037         assert_eq!(events.len(), 1);
1038         let payment_event = SendEvent::from_event(events.pop().unwrap());
1039         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1040         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1041         check_added_monitors!(nodes[1], 1);
1042
1043         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1044         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1045         check_added_monitors!(nodes[0], 1);
1046         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1047         assert_eq!(events.len(), 1);
1048         let payment_event = SendEvent::from_event(events.pop().unwrap());
1049
1050         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1051         check_added_monitors!(nodes[0], 1);
1052         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1053
1054         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1055         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1056         // then restore channel monitor updates.
1057         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1058         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1059         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1060         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1061         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1062         check_added_monitors!(nodes[1], 1);
1063
1064         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1065         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1066         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1067         check_added_monitors!(nodes[1], 1);
1068
1069         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1070         nodes[1].node.test_restore_channel_monitor();
1071         // nodes[1] should be AwaitingRAA here!
1072         check_added_monitors!(nodes[1], 1);
1073         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1074         expect_pending_htlcs_forwardable!(nodes[1]);
1075         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1076
1077         // We send a third payment here, which is somewhat of a redundant test, but the
1078         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1079         // commitment transaction states) whereas here we can explicitly check for it.
1080         nodes[0].node.send_payment(route.clone(), payment_hash_3, None).unwrap();
1081         check_added_monitors!(nodes[0], 0);
1082         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1083
1084         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1085         check_added_monitors!(nodes[0], 1);
1086         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1087         assert_eq!(events.len(), 1);
1088         let payment_event = SendEvent::from_event(events.pop().unwrap());
1089
1090         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1091         check_added_monitors!(nodes[0], 1);
1092         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1093
1094         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1095         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1096         check_added_monitors!(nodes[1], 1);
1097         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1098
1099         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1100         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1101         check_added_monitors!(nodes[1], 1);
1102         expect_pending_htlcs_forwardable!(nodes[1]);
1103         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1104         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1105
1106         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1107         check_added_monitors!(nodes[0], 1);
1108
1109         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1110         check_added_monitors!(nodes[0], 1);
1111         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1112
1113         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1114         check_added_monitors!(nodes[1], 1);
1115         expect_pending_htlcs_forwardable!(nodes[1]);
1116         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1117
1118         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1119         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1120         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1121 }
1122
1123 #[test]
1124 fn claim_while_disconnected_monitor_update_fail() {
1125         // Test for claiming a payment while disconnected and then having the resulting
1126         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1127         // contrived case for nodes with network instability.
1128         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1129         // code introduced a regression in this test (specifically, this caught a removal of the
1130         // channel_reestablish handling ensuring the order was sensical given the messages used).
1131         let node_cfgs = create_node_cfgs(2);
1132         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1133         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1134         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1135
1136         // Forward a payment for B to claim
1137         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1138
1139         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1140         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1141
1142         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1143         check_added_monitors!(nodes[1], 1);
1144
1145         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1146         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1147
1148         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1149         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1150
1151         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1152         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1153
1154         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1155         // update.
1156         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1157
1158         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1159         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1160         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1161         check_added_monitors!(nodes[1], 1);
1162         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1163
1164         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1165         // the monitor still failed
1166         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1167         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1168         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1169         check_added_monitors!(nodes[0], 1);
1170
1171         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1172         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1173         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1174         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1175         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1176         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1177         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1178
1179         // Now un-fail the monitor, which will result in B sending its original commitment update,
1180         // receiving the commitment update from A, and the resulting commitment dances.
1181         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1182         nodes[1].node.test_restore_channel_monitor();
1183         check_added_monitors!(nodes[1], 1);
1184
1185         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1186         assert_eq!(bs_msgs.len(), 2);
1187
1188         match bs_msgs[0] {
1189                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1190                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1191                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1192                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1193                         check_added_monitors!(nodes[0], 1);
1194
1195                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1196                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1197                         check_added_monitors!(nodes[1], 1);
1198                 },
1199                 _ => panic!("Unexpected event"),
1200         }
1201
1202         match bs_msgs[1] {
1203                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1204                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1205                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1206                         check_added_monitors!(nodes[0], 1);
1207                 },
1208                 _ => panic!("Unexpected event"),
1209         }
1210
1211         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1212
1213         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1214         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1215         check_added_monitors!(nodes[0], 1);
1216         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1217
1218         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1219         check_added_monitors!(nodes[1], 1);
1220         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1221         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1222         check_added_monitors!(nodes[1], 1);
1223
1224         expect_pending_htlcs_forwardable!(nodes[1]);
1225         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1226
1227         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1228         check_added_monitors!(nodes[0], 1);
1229
1230         let events = nodes[0].node.get_and_clear_pending_events();
1231         assert_eq!(events.len(), 1);
1232         match events[0] {
1233                 Event::PaymentSent { ref payment_preimage } => {
1234                         assert_eq!(*payment_preimage, payment_preimage_1);
1235                 },
1236                 _ => panic!("Unexpected event"),
1237         }
1238
1239         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1240 }
1241
1242 #[test]
1243 fn monitor_failed_no_reestablish_response() {
1244         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1245         // response to a commitment_signed.
1246         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1247         // debug_assert!() failure in channel_reestablish handling.
1248         let node_cfgs = create_node_cfgs(2);
1249         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1250         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1251         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1252
1253         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1254         // on receipt).
1255         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1256         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1257         nodes[0].node.send_payment(route, payment_hash_1, None).unwrap();
1258         check_added_monitors!(nodes[0], 1);
1259
1260         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1261         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1262         assert_eq!(events.len(), 1);
1263         let payment_event = SendEvent::from_event(events.pop().unwrap());
1264         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1265         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1266         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1267         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1268         check_added_monitors!(nodes[1], 1);
1269
1270         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1271         // is still failing to update monitors.
1272         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1273         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1274
1275         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1276         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1277
1278         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1279         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1280
1281         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1282         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1283
1284         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1285         nodes[1].node.test_restore_channel_monitor();
1286         check_added_monitors!(nodes[1], 1);
1287         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1288
1289         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1290         check_added_monitors!(nodes[0], 1);
1291         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1292         check_added_monitors!(nodes[0], 1);
1293
1294         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1295         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1296         check_added_monitors!(nodes[1], 1);
1297
1298         expect_pending_htlcs_forwardable!(nodes[1]);
1299         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1300
1301         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1302 }
1303
1304 #[test]
1305 fn first_message_on_recv_ordering() {
1306         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1307         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1308         // a commitment_signed which needs to send an RAA first.
1309         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1310         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1311         // response. To do this, we start routing two payments, with the final RAA for the first being
1312         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1313         // have no pending response but will want to send a RAA/CS (with the updates for the second
1314         // payment applied).
1315         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1316         let node_cfgs = create_node_cfgs(2);
1317         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1318         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1319         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1320
1321         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1322         // can deliver it and fail the monitor update.
1323         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1324         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1325         nodes[0].node.send_payment(route, payment_hash_1, None).unwrap();
1326         check_added_monitors!(nodes[0], 1);
1327
1328         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1329         assert_eq!(events.len(), 1);
1330         let payment_event = SendEvent::from_event(events.pop().unwrap());
1331         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1332         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1333         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1334         check_added_monitors!(nodes[1], 1);
1335         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1336
1337         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1338         check_added_monitors!(nodes[0], 1);
1339         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1340         check_added_monitors!(nodes[0], 1);
1341
1342         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1343
1344         // Route the second payment, generating an update_add_htlc/commitment_signed
1345         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1346         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1347         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1348         check_added_monitors!(nodes[0], 1);
1349         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1350         assert_eq!(events.len(), 1);
1351         let payment_event = SendEvent::from_event(events.pop().unwrap());
1352         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1353
1354         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1355
1356         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1357         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1358         // to the next message also tests resetting the delivery order.
1359         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1360         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1361         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1362         check_added_monitors!(nodes[1], 1);
1363
1364         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1365         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1366         // the appropriate HTLC acceptance).
1367         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1368         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1369         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1370         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1371
1372         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1373         nodes[1].node.test_restore_channel_monitor();
1374         check_added_monitors!(nodes[1], 1);
1375
1376         expect_pending_htlcs_forwardable!(nodes[1]);
1377         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1378
1379         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1380         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1381         check_added_monitors!(nodes[0], 1);
1382         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1383         check_added_monitors!(nodes[0], 1);
1384
1385         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1386         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1387         check_added_monitors!(nodes[1], 1);
1388
1389         expect_pending_htlcs_forwardable!(nodes[1]);
1390         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1391
1392         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1393         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1394 }
1395
1396 #[test]
1397 fn test_monitor_update_fail_claim() {
1398         // Basic test for monitor update failures when processing claim_funds calls.
1399         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1400         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1401         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1402         // updating and claim the payment on B.
1403         let node_cfgs = create_node_cfgs(3);
1404         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1405         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1406         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1407         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1408
1409         // Rebalance a bit so that we can send backwards from 3 to 2.
1410         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1411
1412         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1413
1414         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1415         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1416         check_added_monitors!(nodes[1], 1);
1417
1418         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1419         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1420         nodes[2].node.send_payment(route, payment_hash_2, None).unwrap();
1421         check_added_monitors!(nodes[2], 1);
1422
1423         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1424         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1425         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1426
1427         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1428         assert_eq!(events.len(), 1);
1429         let payment_event = SendEvent::from_event(events.pop().unwrap());
1430         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1431         let events = nodes[1].node.get_and_clear_pending_msg_events();
1432         assert_eq!(events.len(), 0);
1433         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1434         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1435
1436         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1437         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1438         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1439
1440         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1441         assert_eq!(msg_events.len(), 1);
1442         match msg_events[0] {
1443                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1444                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1445                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1446                 },
1447                 _ => panic!("Unexpected event"),
1448         }
1449
1450         let events = nodes[2].node.get_and_clear_pending_events();
1451         assert_eq!(events.len(), 1);
1452         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1453                 assert_eq!(payment_hash, payment_hash_2);
1454                 assert!(!rejected_by_dest);
1455         } else { panic!("Unexpected event!"); }
1456
1457         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1458         nodes[1].node.test_restore_channel_monitor();
1459         check_added_monitors!(nodes[1], 1);
1460
1461         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1462         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1463         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1464
1465         let events = nodes[0].node.get_and_clear_pending_events();
1466         assert_eq!(events.len(), 1);
1467         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1468                 assert_eq!(payment_preimage, payment_preimage_1);
1469         } else { panic!("Unexpected event!"); }
1470 }
1471
1472 #[test]
1473 fn test_monitor_update_on_pending_forwards() {
1474         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1475         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1476         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1477         // from C to A will be pending a forward to A.
1478         let node_cfgs = create_node_cfgs(3);
1479         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1480         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1481         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1482         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1483
1484         // Rebalance a bit so that we can send backwards from 3 to 1.
1485         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1486
1487         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1488         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1489         expect_pending_htlcs_forwardable!(nodes[2]);
1490         check_added_monitors!(nodes[2], 1);
1491
1492         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1493         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1494         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1495         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1496
1497         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1498         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1499         nodes[2].node.send_payment(route, payment_hash_2, None).unwrap();
1500         check_added_monitors!(nodes[2], 1);
1501
1502         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1503         assert_eq!(events.len(), 1);
1504         let payment_event = SendEvent::from_event(events.pop().unwrap());
1505         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1506         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1507
1508         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1509         expect_pending_htlcs_forwardable!(nodes[1]);
1510         check_added_monitors!(nodes[1], 1);
1511         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1512         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1513
1514         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1515         nodes[1].node.test_restore_channel_monitor();
1516         check_added_monitors!(nodes[1], 1);
1517
1518         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1519         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1520         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1521         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1522
1523         let events = nodes[0].node.get_and_clear_pending_events();
1524         assert_eq!(events.len(), 2);
1525         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1526                 assert_eq!(payment_hash, payment_hash_1);
1527                 assert!(rejected_by_dest);
1528         } else { panic!("Unexpected event!"); }
1529         match events[1] {
1530                 Event::PendingHTLCsForwardable { .. } => { },
1531                 _ => panic!("Unexpected event"),
1532         };
1533         nodes[0].node.process_pending_htlc_forwards();
1534         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1535
1536         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1537 }
1538
1539 #[test]
1540 fn monitor_update_claim_fail_no_response() {
1541         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1542         // to channel being AwaitingRAA).
1543         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1544         // code was broken.
1545         let node_cfgs = create_node_cfgs(2);
1546         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1547         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1548         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1549
1550         // Forward a payment for B to claim
1551         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1552
1553         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1554         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1555         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1556         nodes[0].node.send_payment(route, payment_hash_2, None).unwrap();
1557         check_added_monitors!(nodes[0], 1);
1558
1559         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1560         assert_eq!(events.len(), 1);
1561         let payment_event = SendEvent::from_event(events.pop().unwrap());
1562         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1563         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1564
1565         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1566         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1567         check_added_monitors!(nodes[1], 1);
1568         let events = nodes[1].node.get_and_clear_pending_msg_events();
1569         assert_eq!(events.len(), 0);
1570         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1571
1572         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1573         nodes[1].node.test_restore_channel_monitor();
1574         check_added_monitors!(nodes[1], 1);
1575         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1576
1577         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1578         check_added_monitors!(nodes[1], 1);
1579         expect_pending_htlcs_forwardable!(nodes[1]);
1580         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1581
1582         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1583         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1584         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1585
1586         let events = nodes[0].node.get_and_clear_pending_events();
1587         assert_eq!(events.len(), 1);
1588         match events[0] {
1589                 Event::PaymentSent { ref payment_preimage } => {
1590                         assert_eq!(*payment_preimage, payment_preimage_1);
1591                 },
1592                 _ => panic!("Unexpected event"),
1593         }
1594
1595         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1596 }
1597
1598 // Note that restore_between_fails with !fail_on_generate is useless
1599 // Also note that !fail_on_generate && !fail_on_signed is useless
1600 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1601 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1602 // restore_b_before_conf has no meaning if !confirm_a_first
1603 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1604         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1605         // the channel setup happily after the update is restored.
1606         let node_cfgs = create_node_cfgs(2);
1607         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1608         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1609
1610         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1611         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1612         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1613
1614         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1615
1616         if fail_on_generate {
1617                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1618         }
1619         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1620         check_added_monitors!(nodes[0], 1);
1621
1622         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1623         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1624         check_added_monitors!(nodes[1], 1);
1625
1626         if restore_between_fails {
1627                 assert!(fail_on_generate);
1628                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1629                 nodes[0].node.test_restore_channel_monitor();
1630                 check_added_monitors!(nodes[0], 1);
1631                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1632                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1633         }
1634
1635         if fail_on_signed {
1636                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1637         } else {
1638                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1639                 assert!(fail_on_generate); // Somebody has to fail
1640         }
1641         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1642         if fail_on_signed || !restore_between_fails {
1643                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1644                 if fail_on_generate && !restore_between_fails {
1645                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1646                         check_added_monitors!(nodes[0], 0);
1647                 } else {
1648                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1649                         check_added_monitors!(nodes[0], 1);
1650                 }
1651                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1652                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1653                 nodes[0].node.test_restore_channel_monitor();
1654         }
1655
1656         check_added_monitors!(nodes[0], 1);
1657
1658         let events = nodes[0].node.get_and_clear_pending_events();
1659         assert_eq!(events.len(), 1);
1660         match events[0] {
1661                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1662                         assert_eq!(user_channel_id, 43);
1663                         assert_eq!(*funding_txo, funding_output);
1664                 },
1665                 _ => panic!("Unexpected event"),
1666         };
1667
1668         if confirm_a_first {
1669                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1670                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1671         } else {
1672                 assert!(!restore_b_before_conf);
1673                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1674                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1675         }
1676
1677         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1678         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1679         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1680         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1681         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1682         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1683
1684         if !restore_b_before_conf {
1685                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1686                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1687                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1688         }
1689
1690         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1691         nodes[1].node.test_restore_channel_monitor();
1692         check_added_monitors!(nodes[1], 1);
1693
1694         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1695                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1696
1697                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1698                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1699                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1700         } else {
1701                 if restore_b_before_conf {
1702                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1703                 }
1704                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1705                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1706         };
1707         for node in nodes.iter() {
1708                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1709                 node.router.handle_channel_update(&as_update).unwrap();
1710                 node.router.handle_channel_update(&bs_update).unwrap();
1711         }
1712
1713         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1714         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1715 }
1716
1717 #[test]
1718 fn during_funding_monitor_fail() {
1719         do_during_funding_monitor_fail(false, false, true, true, true);
1720         do_during_funding_monitor_fail(true, false, true, false, false);
1721         do_during_funding_monitor_fail(true, true, true, true, false);
1722         do_during_funding_monitor_fail(true, true, false, false, false);
1723 }
1724
1725 //TODO: Add a test that creates a channel, but in the inbound direction the monitor update fails to
1726 //persist on funding_signed, but the channelmanager gets persisted, and then when we reload it from
1727 //disk, we need to make sure the channel gets closed!