_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let mut nodes = create_network(2, &[None, None]);
  23         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  24
  25         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  26         let (_, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
  27
  28         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  29         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  30         check_added_monitors!(nodes[0], 1);
  31
  32         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  33         assert_eq!(events_1.len(), 2);
  34         match events_1[0] {
  35                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  36                 _ => panic!("Unexpected event"),
  37         };
  38         match events_1[1] {
  39                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  40                 _ => panic!("Unexpected event"),
  41         };
  42
  43         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  44         // PaymentFailed event
  45
  46         assert_eq!(nodes[0].node.list_channels().len(), 0);
  47 }
  48
  49 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  50         // Test that we can recover from a simple temporary monitor update failure optionally with
  51         // a disconnect in between
  52         let mut nodes = create_network(2, &[None, None]);
  53         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  54
  55         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  56         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
  57
  58         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  59         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  60         check_added_monitors!(nodes[0], 1);
  61
  62         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  63         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  64         assert_eq!(nodes[0].node.list_channels().len(), 1);
  65
  66         if disconnect {
  67                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  68                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  69                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  70         }
  71
  72         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  73         nodes[0].node.test_restore_channel_monitor();
  74         check_added_monitors!(nodes[0], 1);
  75
  76         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  77         assert_eq!(events_2.len(), 1);
  78         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  79         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  80         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  81         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  82
  83         expect_pending_htlcs_forwardable!(nodes[1]);
  84
  85         let events_3 = nodes[1].node.get_and_clear_pending_events();
  86         assert_eq!(events_3.len(), 1);
  87         match events_3[0] {
  88                 Event::PaymentReceived { ref payment_hash, amt } => {
  89                         assert_eq!(payment_hash_1, *payment_hash);
  90                         assert_eq!(amt, 1000000);
  91                 },
  92                 _ => panic!("Unexpected event"),
  93         }
  94
  95         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
  96
  97         // Now set it to failed again...
  98         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
  99         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 100         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 101         check_added_monitors!(nodes[0], 1);
 102
 103         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 104         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 105         assert_eq!(nodes[0].node.list_channels().len(), 1);
 106
 107         if disconnect {
 108                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 109                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 110                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 111         }
 112
 113         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 114         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 115         nodes[0].node.test_restore_channel_monitor();
 116         check_added_monitors!(nodes[0], 1);
 117         check_closed_broadcast!(nodes[0], false);
 118
 119         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 120         // PaymentFailed event
 121
 122         assert_eq!(nodes[0].node.list_channels().len(), 0);
 123 }
 124
 125 #[test]
 126 fn test_simple_monitor_temporary_update_fail() {
 127         do_test_simple_monitor_temporary_update_fail(false);
 128         do_test_simple_monitor_temporary_update_fail(true);
 129 }
 130
 131 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 132         let disconnect_flags = 8 | 16;
 133
 134         // Test that we can recover from a temporary monitor update failure with some in-flight
 135         // HTLCs going on at the same time potentially with some disconnection thrown in.
 136         // * First we route a payment, then get a temporary monitor update failure when trying to
 137         //   route a second payment. We then claim the first payment.
 138         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 139         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 140         //   the ChannelMonitor on a watchtower).
 141         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 142         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 143         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 144         //   disconnect_count & !disconnect_flags is 0).
 145         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 146         //   through message sending, potentially disconnect/reconnecting multiple times based on
 147         //   disconnect_count, to get the update_fulfill_htlc through.
 148         // * We then walk through more message exchanges to get the original update_add_htlc
 149         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 150         //   disconnect/reconnecting based on disconnect_count.
 151         let mut nodes = create_network(2, &[None, None]);
 152         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 153
 154         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 155
 156         // Now try to send a second payment which will fail to send
 157         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 158         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 159
 160         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 161         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 162         check_added_monitors!(nodes[0], 1);
 163
 164         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 165         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 166         assert_eq!(nodes[0].node.list_channels().len(), 1);
 167
 168         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 169         // but nodes[0] won't respond since it is frozen.
 170         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
 171         check_added_monitors!(nodes[1], 1);
 172         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 173         assert_eq!(events_2.len(), 1);
 174         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 175                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 176                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 177                         assert!(update_add_htlcs.is_empty());
 178                         assert_eq!(update_fulfill_htlcs.len(), 1);
 179                         assert!(update_fail_htlcs.is_empty());
 180                         assert!(update_fail_malformed_htlcs.is_empty());
 181                         assert!(update_fee.is_none());
 182
 183                         if (disconnect_count & 16) == 0 {
 184                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 185                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 186                                 assert_eq!(events_3.len(), 1);
 187                                 match events_3[0] {
 188                                         Event::PaymentSent { ref payment_preimage } => {
 189                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 190                                         },
 191                                         _ => panic!("Unexpected event"),
 192                                 }
 193
 194                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 195                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 196                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 197                         }
 198
 199                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 200                 },
 201                 _ => panic!("Unexpected event"),
 202         };
 203
 204         if disconnect_count & !disconnect_flags > 0 {
 205                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 206                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 207         }
 208
 209         // Now fix monitor updating...
 210         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 211         nodes[0].node.test_restore_channel_monitor();
 212         check_added_monitors!(nodes[0], 1);
 213
 214         macro_rules! disconnect_reconnect_peers { () => { {
 215                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 216                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 217
 218                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 219                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 220                 assert_eq!(reestablish_1.len(), 1);
 221                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 222                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 223                 assert_eq!(reestablish_2.len(), 1);
 224
 225                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 226                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 227                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 228                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 229
 230                 assert!(as_resp.0.is_none());
 231                 assert!(bs_resp.0.is_none());
 232
 233                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 234         } } }
 235
 236         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 237                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 238                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 239
 240                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 241                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 242                 assert_eq!(reestablish_1.len(), 1);
 243                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 244                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 245                 assert_eq!(reestablish_2.len(), 1);
 246
 247                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 248                 check_added_monitors!(nodes[0], 0);
 249                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 250                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 251                 check_added_monitors!(nodes[1], 0);
 252                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 253
 254                 assert!(as_resp.0.is_none());
 255                 assert!(bs_resp.0.is_none());
 256
 257                 assert!(bs_resp.1.is_none());
 258                 if (disconnect_count & 16) == 0 {
 259                         assert!(bs_resp.2.is_none());
 260
 261                         assert!(as_resp.1.is_some());
 262                         assert!(as_resp.2.is_some());
 263                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 264                 } else {
 265                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 266                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 267                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 268                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 269                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 270                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 271
 272                         assert!(as_resp.1.is_none());
 273
 274                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 275                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 276                         assert_eq!(events_3.len(), 1);
 277                         match events_3[0] {
 278                                 Event::PaymentSent { ref payment_preimage } => {
 279                                         assert_eq!(*payment_preimage, payment_preimage_1);
 280                                 },
 281                                 _ => panic!("Unexpected event"),
 282                         }
 283
 284                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 285                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 286                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 287                         check_added_monitors!(nodes[0], 1);
 288
 289                         as_resp.1 = Some(as_resp_raa);
 290                         bs_resp.2 = None;
 291                 }
 292
 293                 if disconnect_count & !disconnect_flags > 1 {
 294                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 295
 296                         if (disconnect_count & 16) == 0 {
 297                                 assert!(reestablish_1 == second_reestablish_1);
 298                                 assert!(reestablish_2 == second_reestablish_2);
 299                         }
 300                         assert!(as_resp == second_as_resp);
 301                         assert!(bs_resp == second_bs_resp);
 302                 }
 303
 304                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 305         } else {
 306                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 307                 assert_eq!(events_4.len(), 2);
 308                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 309                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 310                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 311                                 msg.clone()
 312                         },
 313                         _ => panic!("Unexpected event"),
 314                 })
 315         };
 316
 317         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 318
 319         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 320         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 321         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 322         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 323         check_added_monitors!(nodes[1], 1);
 324
 325         if disconnect_count & !disconnect_flags > 2 {
 326                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 327
 328                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 329                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 330
 331                 assert!(as_resp.2.is_none());
 332                 assert!(bs_resp.2.is_none());
 333         }
 334
 335         let as_commitment_update;
 336         let bs_second_commitment_update;
 337
 338         macro_rules! handle_bs_raa { () => {
 339                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 340                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 341                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 342                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 343                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 344                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 345                 assert!(as_commitment_update.update_fee.is_none());
 346                 check_added_monitors!(nodes[0], 1);
 347         } }
 348
 349         macro_rules! handle_initial_raa { () => {
 350                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 351                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 352                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 353                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 354                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 355                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 356                 assert!(bs_second_commitment_update.update_fee.is_none());
 357                 check_added_monitors!(nodes[1], 1);
 358         } }
 359
 360         if (disconnect_count & 8) == 0 {
 361                 handle_bs_raa!();
 362
 363                 if disconnect_count & !disconnect_flags > 3 {
 364                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 365
 366                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 367                         assert!(bs_resp.1.is_none());
 368
 369                         assert!(as_resp.2.unwrap() == as_commitment_update);
 370                         assert!(bs_resp.2.is_none());
 371
 372                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 373                 }
 374
 375                 handle_initial_raa!();
 376
 377                 if disconnect_count & !disconnect_flags > 4 {
 378                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 379
 380                         assert!(as_resp.1.is_none());
 381                         assert!(bs_resp.1.is_none());
 382
 383                         assert!(as_resp.2.unwrap() == as_commitment_update);
 384                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 385                 }
 386         } else {
 387                 handle_initial_raa!();
 388
 389                 if disconnect_count & !disconnect_flags > 3 {
 390                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 391
 392                         assert!(as_resp.1.is_none());
 393                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 394
 395                         assert!(as_resp.2.is_none());
 396                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 397
 398                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 399                 }
 400
 401                 handle_bs_raa!();
 402
 403                 if disconnect_count & !disconnect_flags > 4 {
 404                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 405
 406                         assert!(as_resp.1.is_none());
 407                         assert!(bs_resp.1.is_none());
 408
 409                         assert!(as_resp.2.unwrap() == as_commitment_update);
 410                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 411                 }
 412         }
 413
 414         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 415         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 416         // No commitment_signed so get_event_msg's assert(len == 1) passes
 417         check_added_monitors!(nodes[0], 1);
 418
 419         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 420         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 421         // No commitment_signed so get_event_msg's assert(len == 1) passes
 422         check_added_monitors!(nodes[1], 1);
 423
 424         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 425         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 426         check_added_monitors!(nodes[1], 1);
 427
 428         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 429         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 430         check_added_monitors!(nodes[0], 1);
 431
 432         expect_pending_htlcs_forwardable!(nodes[1]);
 433
 434         let events_5 = nodes[1].node.get_and_clear_pending_events();
 435         assert_eq!(events_5.len(), 1);
 436         match events_5[0] {
 437                 Event::PaymentReceived { ref payment_hash, amt } => {
 438                         assert_eq!(payment_hash_2, *payment_hash);
 439                         assert_eq!(amt, 1000000);
 440                 },
 441                 _ => panic!("Unexpected event"),
 442         }
 443
 444         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 445 }
 446
 447 #[test]
 448 fn test_monitor_temporary_update_fail_a() {
 449         do_test_monitor_temporary_update_fail(0);
 450         do_test_monitor_temporary_update_fail(1);
 451         do_test_monitor_temporary_update_fail(2);
 452         do_test_monitor_temporary_update_fail(3);
 453         do_test_monitor_temporary_update_fail(4);
 454         do_test_monitor_temporary_update_fail(5);
 455 }
 456
 457 #[test]
 458 fn test_monitor_temporary_update_fail_b() {
 459         do_test_monitor_temporary_update_fail(2 | 8);
 460         do_test_monitor_temporary_update_fail(3 | 8);
 461         do_test_monitor_temporary_update_fail(4 | 8);
 462         do_test_monitor_temporary_update_fail(5 | 8);
 463 }
 464
 465 #[test]
 466 fn test_monitor_temporary_update_fail_c() {
 467         do_test_monitor_temporary_update_fail(1 | 16);
 468         do_test_monitor_temporary_update_fail(2 | 16);
 469         do_test_monitor_temporary_update_fail(3 | 16);
 470         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 471         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 472 }
 473
 474 #[test]
 475 fn test_monitor_update_fail_cs() {
 476         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 477         let mut nodes = create_network(2, &[None, None]);
 478         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 479
 480         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 481         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 482         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 483         check_added_monitors!(nodes[0], 1);
 484
 485         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 486         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 487
 488         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 489         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 490         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 491         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 492         check_added_monitors!(nodes[1], 1);
 493         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 494
 495         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 496         nodes[1].node.test_restore_channel_monitor();
 497         check_added_monitors!(nodes[1], 1);
 498         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 499         assert_eq!(responses.len(), 2);
 500
 501         match responses[0] {
 502                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 503                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 504                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 505                         check_added_monitors!(nodes[0], 1);
 506                 },
 507                 _ => panic!("Unexpected event"),
 508         }
 509         match responses[1] {
 510                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 511                         assert!(updates.update_add_htlcs.is_empty());
 512                         assert!(updates.update_fulfill_htlcs.is_empty());
 513                         assert!(updates.update_fail_htlcs.is_empty());
 514                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 515                         assert!(updates.update_fee.is_none());
 516                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 517
 518                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 519                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 520                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 521                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 522                         check_added_monitors!(nodes[0], 1);
 523                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 524                 },
 525                 _ => panic!("Unexpected event"),
 526         }
 527
 528         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 529         nodes[0].node.test_restore_channel_monitor();
 530         check_added_monitors!(nodes[0], 1);
 531
 532         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 533         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 534         check_added_monitors!(nodes[1], 1);
 535
 536         expect_pending_htlcs_forwardable!(nodes[1]);
 537
 538         let events = nodes[1].node.get_and_clear_pending_events();
 539         assert_eq!(events.len(), 1);
 540         match events[0] {
 541                 Event::PaymentReceived { payment_hash, amt } => {
 542                         assert_eq!(payment_hash, our_payment_hash);
 543                         assert_eq!(amt, 1000000);
 544                 },
 545                 _ => panic!("Unexpected event"),
 546         };
 547
 548         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 549 }
 550
 551 #[test]
 552 fn test_monitor_update_fail_no_rebroadcast() {
 553         // Tests handling of a monitor update failure when no message rebroadcasting on
 554         // test_restore_channel_monitor() is required. Backported from
 555         // chanmon_fail_consistency fuzz tests.
 556         let mut nodes = create_network(2, &[None, None]);
 557         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 558
 559         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 560         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 561         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 562         check_added_monitors!(nodes[0], 1);
 563
 564         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 565         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 566         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 567
 568         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 569         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 570         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 571         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 572         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 573         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 574         check_added_monitors!(nodes[1], 1);
 575
 576         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 577         nodes[1].node.test_restore_channel_monitor();
 578         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 579         check_added_monitors!(nodes[1], 1);
 580         expect_pending_htlcs_forwardable!(nodes[1]);
 581
 582         let events = nodes[1].node.get_and_clear_pending_events();
 583         assert_eq!(events.len(), 1);
 584         match events[0] {
 585                 Event::PaymentReceived { payment_hash, .. } => {
 586                         assert_eq!(payment_hash, our_payment_hash);
 587                 },
 588                 _ => panic!("Unexpected event"),
 589         }
 590
 591         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 592 }
 593
 594 #[test]
 595 fn test_monitor_update_raa_while_paused() {
 596         // Tests handling of an RAA while monitor updating has already been marked failed.
 597         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 598         let mut nodes = create_network(2, &[None, None]);
 599         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 600
 601         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 602
 603         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 604         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 605         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 606         check_added_monitors!(nodes[0], 1);
 607         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 608
 609         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 610         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 611         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 612         check_added_monitors!(nodes[1], 1);
 613         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 614
 615         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 616         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 617         check_added_monitors!(nodes[1], 1);
 618         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 619
 620         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 621         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 622         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 623         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 624         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 625         check_added_monitors!(nodes[0], 1);
 626
 627         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 628         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 629         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 630         check_added_monitors!(nodes[0], 1);
 631
 632         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 633         nodes[0].node.test_restore_channel_monitor();
 634         check_added_monitors!(nodes[0], 1);
 635
 636         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 637         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 638         check_added_monitors!(nodes[1], 1);
 639         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 640
 641         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 642         check_added_monitors!(nodes[1], 1);
 643         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 644
 645         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 646         check_added_monitors!(nodes[0], 1);
 647         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 648
 649         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 650         check_added_monitors!(nodes[0], 1);
 651         expect_pending_htlcs_forwardable!(nodes[0]);
 652         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 653
 654         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 655         check_added_monitors!(nodes[1], 1);
 656         expect_pending_htlcs_forwardable!(nodes[1]);
 657         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 658
 659         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 660         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 661 }
 662
 663 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 664         // Tests handling of a monitor update failure when processing an incoming RAA
 665         let mut nodes = create_network(3, &[None, None, None]);
 666         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 667         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 668
 669         // Rebalance a bit so that we can send backwards from 2 to 1.
 670         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 671
 672         // Route a first payment that we'll fail backwards
 673         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 674
 675         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 676         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 677         expect_pending_htlcs_forwardable!(nodes[2]);
 678         check_added_monitors!(nodes[2], 1);
 679
 680         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 681         assert!(updates.update_add_htlcs.is_empty());
 682         assert!(updates.update_fulfill_htlcs.is_empty());
 683         assert_eq!(updates.update_fail_htlcs.len(), 1);
 684         assert!(updates.update_fail_malformed_htlcs.is_empty());
 685         assert!(updates.update_fee.is_none());
 686         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 687
 688         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 689         check_added_monitors!(nodes[0], 0);
 690
 691         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 692         // holding cell.
 693         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 694         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 695         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 696         check_added_monitors!(nodes[0], 1);
 697
 698         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 699         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 700         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 701
 702         expect_pending_htlcs_forwardable!(nodes[1]);
 703         check_added_monitors!(nodes[1], 0);
 704         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 705
 706         // Now fail monitor updating.
 707         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 708         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 709         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 710         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 711         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 712         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 713         check_added_monitors!(nodes[1], 1);
 714
 715         // Attempt to forward a third payment but fail due to the second channel being unavailable
 716         // for forwarding.
 717
 718         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 719         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 720         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 721         check_added_monitors!(nodes[0], 1);
 722
 723         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 724         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 725         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 726         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 727         check_added_monitors!(nodes[1], 0);
 728
 729         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 730         assert_eq!(events_2.len(), 1);
 731         match events_2.remove(0) {
 732                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 733                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 734                         assert!(updates.update_fulfill_htlcs.is_empty());
 735                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 736                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 737                         assert!(updates.update_add_htlcs.is_empty());
 738                         assert!(updates.update_fee.is_none());
 739
 740                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 741                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 742
 743                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 744                         assert_eq!(msg_events.len(), 1);
 745                         match msg_events[0] {
 746                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 747                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 748                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 749                                 },
 750                                 _ => panic!("Unexpected event"),
 751                         }
 752
 753                         let events = nodes[0].node.get_and_clear_pending_events();
 754                         assert_eq!(events.len(), 1);
 755                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 756                                 assert_eq!(payment_hash, payment_hash_3);
 757                                 assert!(!rejected_by_dest);
 758                         } else { panic!("Unexpected event!"); }
 759                 },
 760                 _ => panic!("Unexpected event type!"),
 761         };
 762
 763         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 764                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 765                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 766                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 767                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 768                 check_added_monitors!(nodes[2], 1);
 769
 770                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 771                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 772                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 773                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 774                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 775                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 776                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 777                 (Some(payment_preimage_4), Some(payment_hash_4))
 778         } else { (None, None) };
 779
 780         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 781         // update_add update.
 782         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 783         nodes[1].node.test_restore_channel_monitor();
 784         check_added_monitors!(nodes[1], 1);
 785         expect_pending_htlcs_forwardable!(nodes[1]);
 786         check_added_monitors!(nodes[1], 1);
 787
 788         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 789         if test_ignore_second_cs {
 790                 assert_eq!(events_3.len(), 3);
 791         } else {
 792                 assert_eq!(events_3.len(), 2);
 793         }
 794
 795         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 796         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 797         let messages_a = match events_3.pop().unwrap() {
 798                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 799                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 800                         assert!(updates.update_fulfill_htlcs.is_empty());
 801                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 802                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 803                         assert!(updates.update_add_htlcs.is_empty());
 804                         assert!(updates.update_fee.is_none());
 805                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 806                 },
 807                 _ => panic!("Unexpected event type!"),
 808         };
 809         let raa = if test_ignore_second_cs {
 810                 match events_3.remove(1) {
 811                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 812                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 813                                 Some(msg.clone())
 814                         },
 815                         _ => panic!("Unexpected event"),
 816                 }
 817         } else { None };
 818         let send_event_b = SendEvent::from_event(events_3.remove(0));
 819         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 820
 821         // Now deliver the new messages...
 822
 823         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 824         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 825         let events_4 = nodes[0].node.get_and_clear_pending_events();
 826         assert_eq!(events_4.len(), 1);
 827         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 828                 assert_eq!(payment_hash, payment_hash_1);
 829                 assert!(rejected_by_dest);
 830         } else { panic!("Unexpected event!"); }
 831
 832         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 833         if test_ignore_second_cs {
 834                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 835                 check_added_monitors!(nodes[2], 1);
 836                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 837                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 838                 check_added_monitors!(nodes[2], 1);
 839                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 840                 assert!(bs_cs.update_add_htlcs.is_empty());
 841                 assert!(bs_cs.update_fail_htlcs.is_empty());
 842                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 843                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 844                 assert!(bs_cs.update_fee.is_none());
 845
 846                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 847                 check_added_monitors!(nodes[1], 1);
 848                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 849                 assert!(as_cs.update_add_htlcs.is_empty());
 850                 assert!(as_cs.update_fail_htlcs.is_empty());
 851                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 852                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 853                 assert!(as_cs.update_fee.is_none());
 854
 855                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 856                 check_added_monitors!(nodes[1], 1);
 857                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 858
 859                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 860                 check_added_monitors!(nodes[2], 1);
 861                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 862
 863                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 864                 check_added_monitors!(nodes[2], 1);
 865                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 866
 867                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 868                 check_added_monitors!(nodes[1], 1);
 869                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 870         } else {
 871                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 872         }
 873
 874         expect_pending_htlcs_forwardable!(nodes[2]);
 875
 876         let events_6 = nodes[2].node.get_and_clear_pending_events();
 877         assert_eq!(events_6.len(), 1);
 878         match events_6[0] {
 879                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 880                 _ => panic!("Unexpected event"),
 881         };
 882
 883         if test_ignore_second_cs {
 884                 expect_pending_htlcs_forwardable!(nodes[1]);
 885                 check_added_monitors!(nodes[1], 1);
 886
 887                 send_event = SendEvent::from_node(&nodes[1]);
 888                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 889                 assert_eq!(send_event.msgs.len(), 1);
 890                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 891                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 892
 893                 expect_pending_htlcs_forwardable!(nodes[0]);
 894
 895                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 896                 assert_eq!(events_9.len(), 1);
 897                 match events_9[0] {
 898                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 899                         _ => panic!("Unexpected event"),
 900                 };
 901                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 902         }
 903
 904         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 905 }
 906
 907 #[test]
 908 fn test_monitor_update_fail_raa() {
 909         do_test_monitor_update_fail_raa(false);
 910         do_test_monitor_update_fail_raa(true);
 911 }
 912
 913 #[test]
 914 fn test_monitor_update_fail_reestablish() {
 915         // Simple test for message retransmission after monitor update failure on
 916         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 917         // HTLCs).
 918         let mut nodes = create_network(3, &[None, None, None]);
 919         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 920         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 921
 922         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 923
 924         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 925         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 926
 927         assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
 928         check_added_monitors!(nodes[2], 1);
 929         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 930         assert!(updates.update_add_htlcs.is_empty());
 931         assert!(updates.update_fail_htlcs.is_empty());
 932         assert!(updates.update_fail_malformed_htlcs.is_empty());
 933         assert!(updates.update_fee.is_none());
 934         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 935         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 936         check_added_monitors!(nodes[1], 1);
 937         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 938         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 939
 940         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 941         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 942         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 943
 944         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 945         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 946
 947         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 948
 949         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 950         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 951         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 952         check_added_monitors!(nodes[1], 1);
 953
 954         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 955         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 956
 957         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 958         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 959
 960         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 961         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 962
 963         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 964
 965         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 966         check_added_monitors!(nodes[1], 0);
 967         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 968
 969         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 970         nodes[1].node.test_restore_channel_monitor();
 971         check_added_monitors!(nodes[1], 1);
 972
 973         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 974         assert!(updates.update_add_htlcs.is_empty());
 975         assert!(updates.update_fail_htlcs.is_empty());
 976         assert!(updates.update_fail_malformed_htlcs.is_empty());
 977         assert!(updates.update_fee.is_none());
 978         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 979         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 980         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
 981
 982         let events = nodes[0].node.get_and_clear_pending_events();
 983         assert_eq!(events.len(), 1);
 984         match events[0] {
 985                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
 986                 _ => panic!("Unexpected event"),
 987         }
 988 }
 989
 990 #[test]
 991 fn raa_no_response_awaiting_raa_state() {
 992         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
 993         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
 994         // in question (assuming it intends to respond with a CS after monitor updating is restored).
 995         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 996         let mut nodes = create_network(2, &[None, None]);
 997         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 998
 999         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1000         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1001         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1002         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1003
1004         // Queue up two payments - one will be delivered right away, one immediately goes into the
1005         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1006         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1007         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1008         // generation during RAA while in monitor-update-failed state.
1009         nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1010         check_added_monitors!(nodes[0], 1);
1011         nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1012         check_added_monitors!(nodes[0], 0);
1013
1014         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1015         assert_eq!(events.len(), 1);
1016         let payment_event = SendEvent::from_event(events.pop().unwrap());
1017         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1018         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1019         check_added_monitors!(nodes[1], 1);
1020
1021         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1022         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1023         check_added_monitors!(nodes[0], 1);
1024         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1025         assert_eq!(events.len(), 1);
1026         let payment_event = SendEvent::from_event(events.pop().unwrap());
1027
1028         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1029         check_added_monitors!(nodes[0], 1);
1030         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1031
1032         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1033         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1034         // then restore channel monitor updates.
1035         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1036         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1037         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1038         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1039         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1040         check_added_monitors!(nodes[1], 1);
1041
1042         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1043         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1044         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1045         check_added_monitors!(nodes[1], 1);
1046
1047         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1048         nodes[1].node.test_restore_channel_monitor();
1049         // nodes[1] should be AwaitingRAA here!
1050         check_added_monitors!(nodes[1], 1);
1051         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1052         expect_pending_htlcs_forwardable!(nodes[1]);
1053         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1054
1055         // We send a third payment here, which is somewhat of a redundant test, but the
1056         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1057         // commitment transaction states) whereas here we can explicitly check for it.
1058         nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1059         check_added_monitors!(nodes[0], 0);
1060         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1061
1062         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1063         check_added_monitors!(nodes[0], 1);
1064         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1065         assert_eq!(events.len(), 1);
1066         let payment_event = SendEvent::from_event(events.pop().unwrap());
1067
1068         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1069         check_added_monitors!(nodes[0], 1);
1070         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1071
1072         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1073         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1074         check_added_monitors!(nodes[1], 1);
1075         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1076
1077         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1078         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1079         check_added_monitors!(nodes[1], 1);
1080         expect_pending_htlcs_forwardable!(nodes[1]);
1081         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1082         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1083
1084         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1085         check_added_monitors!(nodes[0], 1);
1086
1087         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1088         check_added_monitors!(nodes[0], 1);
1089         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1090
1091         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1092         check_added_monitors!(nodes[1], 1);
1093         expect_pending_htlcs_forwardable!(nodes[1]);
1094         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1095
1096         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1097         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1098         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1099 }
1100
1101 #[test]
1102 fn claim_while_disconnected_monitor_update_fail() {
1103         // Test for claiming a payment while disconnected and then having the resulting
1104         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1105         // contrived case for nodes with network instability.
1106         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1107         // code introduced a regression in this test (specifically, this caught a removal of the
1108         // channel_reestablish handling ensuring the order was sensical given the messages used).
1109         let mut nodes = create_network(2, &[None, None]);
1110         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1111
1112         // Forward a payment for B to claim
1113         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1114
1115         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1116         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1117
1118         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1119         check_added_monitors!(nodes[1], 1);
1120
1121         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1122         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1123
1124         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1125         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1126
1127         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1128         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1129
1130         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1131         // update.
1132         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1133
1134         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1135         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1136         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1137         check_added_monitors!(nodes[1], 1);
1138         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1139
1140         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1141         // the monitor still failed
1142         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1143         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1144         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1145         check_added_monitors!(nodes[0], 1);
1146
1147         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1148         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1149         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1150         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1151         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1152         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1153         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1154
1155         // Now un-fail the monitor, which will result in B sending its original commitment update,
1156         // receiving the commitment update from A, and the resulting commitment dances.
1157         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1158         nodes[1].node.test_restore_channel_monitor();
1159         check_added_monitors!(nodes[1], 1);
1160
1161         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1162         assert_eq!(bs_msgs.len(), 2);
1163
1164         match bs_msgs[0] {
1165                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1166                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1167                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1168                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1169                         check_added_monitors!(nodes[0], 1);
1170
1171                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1172                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1173                         check_added_monitors!(nodes[1], 1);
1174                 },
1175                 _ => panic!("Unexpected event"),
1176         }
1177
1178         match bs_msgs[1] {
1179                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1180                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1181                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1182                         check_added_monitors!(nodes[0], 1);
1183                 },
1184                 _ => panic!("Unexpected event"),
1185         }
1186
1187         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1188
1189         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1190         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1191         check_added_monitors!(nodes[0], 1);
1192         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1193
1194         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1195         check_added_monitors!(nodes[1], 1);
1196         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1197         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1198         check_added_monitors!(nodes[1], 1);
1199
1200         expect_pending_htlcs_forwardable!(nodes[1]);
1201         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1202
1203         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1204         check_added_monitors!(nodes[0], 1);
1205
1206         let events = nodes[0].node.get_and_clear_pending_events();
1207         assert_eq!(events.len(), 1);
1208         match events[0] {
1209                 Event::PaymentSent { ref payment_preimage } => {
1210                         assert_eq!(*payment_preimage, payment_preimage_1);
1211                 },
1212                 _ => panic!("Unexpected event"),
1213         }
1214
1215         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1216 }
1217
1218 #[test]
1219 fn monitor_failed_no_reestablish_response() {
1220         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1221         // response to a commitment_signed.
1222         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1223         // debug_assert!() failure in channel_reestablish handling.
1224         let mut nodes = create_network(2, &[None, None]);
1225         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1226
1227         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1228         // on receipt).
1229         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1230         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1231         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1232         check_added_monitors!(nodes[0], 1);
1233
1234         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1235         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1236         assert_eq!(events.len(), 1);
1237         let payment_event = SendEvent::from_event(events.pop().unwrap());
1238         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1239         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1240         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1241         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1242         check_added_monitors!(nodes[1], 1);
1243
1244         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1245         // is still failing to update monitors.
1246         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1247         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1248
1249         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1250         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1251
1252         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1253         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1254
1255         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1256         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1257
1258         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1259         nodes[1].node.test_restore_channel_monitor();
1260         check_added_monitors!(nodes[1], 1);
1261         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1262
1263         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1264         check_added_monitors!(nodes[0], 1);
1265         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1266         check_added_monitors!(nodes[0], 1);
1267
1268         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1269         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1270         check_added_monitors!(nodes[1], 1);
1271
1272         expect_pending_htlcs_forwardable!(nodes[1]);
1273         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1274
1275         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1276 }
1277
1278 #[test]
1279 fn first_message_on_recv_ordering() {
1280         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1281         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1282         // a commitment_signed which needs to send an RAA first.
1283         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1284         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1285         // response. To do this, we start routing two payments, with the final RAA for the first being
1286         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1287         // have no pending response but will want to send a RAA/CS (with the updates for the second
1288         // payment applied).
1289         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1290         let mut nodes = create_network(2, &[None, None]);
1291         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1292
1293         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1294         // can deliver it and fail the monitor update.
1295         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1296         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1297         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1298         check_added_monitors!(nodes[0], 1);
1299
1300         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1301         assert_eq!(events.len(), 1);
1302         let payment_event = SendEvent::from_event(events.pop().unwrap());
1303         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1304         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1305         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1306         check_added_monitors!(nodes[1], 1);
1307         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1308
1309         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1310         check_added_monitors!(nodes[0], 1);
1311         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1312         check_added_monitors!(nodes[0], 1);
1313
1314         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1315
1316         // Route the second payment, generating an update_add_htlc/commitment_signed
1317         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1318         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1319         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1320         check_added_monitors!(nodes[0], 1);
1321         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1322         assert_eq!(events.len(), 1);
1323         let payment_event = SendEvent::from_event(events.pop().unwrap());
1324         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1325
1326         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1327
1328         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1329         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1330         // to the next message also tests resetting the delivery order.
1331         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1332         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1333         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1334         check_added_monitors!(nodes[1], 1);
1335
1336         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1337         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1338         // the appropriate HTLC acceptance).
1339         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1340         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1341         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1342         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1343
1344         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1345         nodes[1].node.test_restore_channel_monitor();
1346         check_added_monitors!(nodes[1], 1);
1347
1348         expect_pending_htlcs_forwardable!(nodes[1]);
1349         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1350
1351         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1352         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1353         check_added_monitors!(nodes[0], 1);
1354         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1355         check_added_monitors!(nodes[0], 1);
1356
1357         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1358         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1359         check_added_monitors!(nodes[1], 1);
1360
1361         expect_pending_htlcs_forwardable!(nodes[1]);
1362         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1363
1364         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1365         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1366 }
1367
1368 #[test]
1369 fn test_monitor_update_fail_claim() {
1370         // Basic test for monitor update failures when processing claim_funds calls.
1371         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1372         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1373         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1374         // updating and claim the payment on B.
1375         let mut nodes = create_network(3, &[None, None, None]);
1376         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1377         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1378
1379         // Rebalance a bit so that we can send backwards from 3 to 2.
1380         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1381
1382         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1383
1384         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1385         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1386         check_added_monitors!(nodes[1], 1);
1387
1388         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1389         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1390         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1391         check_added_monitors!(nodes[2], 1);
1392
1393         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1394         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1395         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1396
1397         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1398         assert_eq!(events.len(), 1);
1399         let payment_event = SendEvent::from_event(events.pop().unwrap());
1400         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1401         let events = nodes[1].node.get_and_clear_pending_msg_events();
1402         assert_eq!(events.len(), 0);
1403         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1404         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1405
1406         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1407         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1408         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1409
1410         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1411         assert_eq!(msg_events.len(), 1);
1412         match msg_events[0] {
1413                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1414                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1415                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1416                 },
1417                 _ => panic!("Unexpected event"),
1418         }
1419
1420         let events = nodes[2].node.get_and_clear_pending_events();
1421         assert_eq!(events.len(), 1);
1422         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1423                 assert_eq!(payment_hash, payment_hash_2);
1424                 assert!(!rejected_by_dest);
1425         } else { panic!("Unexpected event!"); }
1426
1427         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1428         nodes[1].node.test_restore_channel_monitor();
1429         check_added_monitors!(nodes[1], 1);
1430
1431         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1432         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1433         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1434
1435         let events = nodes[0].node.get_and_clear_pending_events();
1436         assert_eq!(events.len(), 1);
1437         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1438                 assert_eq!(payment_preimage, payment_preimage_1);
1439         } else { panic!("Unexpected event!"); }
1440 }
1441
1442 #[test]
1443 fn test_monitor_update_on_pending_forwards() {
1444         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1445         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1446         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1447         // from C to A will be pending a forward to A.
1448         let mut nodes = create_network(3, &[None, None, None]);
1449         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1450         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1451
1452         // Rebalance a bit so that we can send backwards from 3 to 1.
1453         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1454
1455         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1456         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1457         expect_pending_htlcs_forwardable!(nodes[2]);
1458         check_added_monitors!(nodes[2], 1);
1459
1460         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1461         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1462         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1463         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1464
1465         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1466         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1467         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1468         check_added_monitors!(nodes[2], 1);
1469
1470         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1471         assert_eq!(events.len(), 1);
1472         let payment_event = SendEvent::from_event(events.pop().unwrap());
1473         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1474         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1475
1476         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1477         expect_pending_htlcs_forwardable!(nodes[1]);
1478         check_added_monitors!(nodes[1], 1);
1479         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1480         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1481
1482         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1483         nodes[1].node.test_restore_channel_monitor();
1484         check_added_monitors!(nodes[1], 1);
1485
1486         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1487         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1488         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1489         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1490
1491         let events = nodes[0].node.get_and_clear_pending_events();
1492         assert_eq!(events.len(), 2);
1493         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1494                 assert_eq!(payment_hash, payment_hash_1);
1495                 assert!(rejected_by_dest);
1496         } else { panic!("Unexpected event!"); }
1497         match events[1] {
1498                 Event::PendingHTLCsForwardable { .. } => { },
1499                 _ => panic!("Unexpected event"),
1500         };
1501         nodes[0].node.process_pending_htlc_forwards();
1502         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1503
1504         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1505 }
1506
1507 #[test]
1508 fn monitor_update_claim_fail_no_response() {
1509         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1510         // to channel being AwaitingRAA).
1511         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1512         // code was broken.
1513         let mut nodes = create_network(2, &[None, None]);
1514         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1515
1516         // Forward a payment for B to claim
1517         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1518
1519         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1520         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1521         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1522         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1523         check_added_monitors!(nodes[0], 1);
1524
1525         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1526         assert_eq!(events.len(), 1);
1527         let payment_event = SendEvent::from_event(events.pop().unwrap());
1528         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1529         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1530
1531         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1532         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1533         check_added_monitors!(nodes[1], 1);
1534         let events = nodes[1].node.get_and_clear_pending_msg_events();
1535         assert_eq!(events.len(), 0);
1536         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1537
1538         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1539         nodes[1].node.test_restore_channel_monitor();
1540         check_added_monitors!(nodes[1], 1);
1541         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1542
1543         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1544         check_added_monitors!(nodes[1], 1);
1545         expect_pending_htlcs_forwardable!(nodes[1]);
1546         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1547
1548         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1549         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1550         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1551
1552         let events = nodes[0].node.get_and_clear_pending_events();
1553         assert_eq!(events.len(), 1);
1554         match events[0] {
1555                 Event::PaymentSent { ref payment_preimage } => {
1556                         assert_eq!(*payment_preimage, payment_preimage_1);
1557                 },
1558                 _ => panic!("Unexpected event"),
1559         }
1560
1561         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1562 }
1563
1564 // Note that restore_between_fails with !fail_on_generate is useless
1565 // Also note that !fail_on_generate && !fail_on_signed is useless
1566 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1567 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1568 // restore_b_before_conf has no meaning if !confirm_a_first
1569 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1570         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1571         // the channel setup happily after the update is restored.
1572         let mut nodes = create_network(2, &[None, None]);
1573
1574         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1575         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1576         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1577
1578         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1579
1580         if fail_on_generate {
1581                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1582         }
1583         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1584         check_added_monitors!(nodes[0], 1);
1585
1586         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1587         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1588         check_added_monitors!(nodes[1], 1);
1589
1590         if restore_between_fails {
1591                 assert!(fail_on_generate);
1592                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1593                 nodes[0].node.test_restore_channel_monitor();
1594                 check_added_monitors!(nodes[0], 1);
1595                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1596                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1597         }
1598
1599         if fail_on_signed {
1600                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1601         } else {
1602                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1603                 assert!(fail_on_generate); // Somebody has to fail
1604         }
1605         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1606         if fail_on_signed || !restore_between_fails {
1607                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1608                 if fail_on_generate && !restore_between_fails {
1609                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1610                         check_added_monitors!(nodes[0], 0);
1611                 } else {
1612                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1613                         check_added_monitors!(nodes[0], 1);
1614                 }
1615                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1616                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1617                 nodes[0].node.test_restore_channel_monitor();
1618         }
1619
1620         check_added_monitors!(nodes[0], 1);
1621
1622         let events = nodes[0].node.get_and_clear_pending_events();
1623         assert_eq!(events.len(), 1);
1624         match events[0] {
1625                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1626                         assert_eq!(user_channel_id, 43);
1627                         assert_eq!(*funding_txo, funding_output);
1628                 },
1629                 _ => panic!("Unexpected event"),
1630         };
1631
1632         if confirm_a_first {
1633                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1634                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1635         } else {
1636                 assert!(!restore_b_before_conf);
1637                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1638                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1639         }
1640
1641         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1642         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1643         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1644         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1645         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1646         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1647
1648         if !restore_b_before_conf {
1649                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1650                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1651                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1652         }
1653
1654         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1655         nodes[1].node.test_restore_channel_monitor();
1656         check_added_monitors!(nodes[1], 1);
1657
1658         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1659                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1660
1661                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1662                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1663                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1664         } else {
1665                 if restore_b_before_conf {
1666                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1667                 }
1668                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1669                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1670         };
1671         for node in nodes.iter() {
1672                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1673                 node.router.handle_channel_update(&as_update).unwrap();
1674                 node.router.handle_channel_update(&bs_update).unwrap();
1675         }
1676
1677         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1678         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1679 }
1680
1681 #[test]
1682 fn during_funding_monitor_fail() {
1683         do_during_funding_monitor_fail(false, false, true, true, true);
1684         do_during_funding_monitor_fail(true, false, true, false, false);
1685         do_during_funding_monitor_fail(true, true, true, true, false);
1686         do_during_funding_monitor_fail(true, true, false, false, false);
1687 }