_ Git - rust-lightning/blob - src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::msgs;
   9 use ln::msgs::ChannelMessageHandler;
  10 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  11 use util::errors::APIError;
  12
  13 use bitcoin_hashes::sha256::Hash as Sha256;
  14 use bitcoin_hashes::Hash;
  15
  16 use std::time::Instant;
  17
  18 use ln::functional_test_utils::*;
  19
  20 #[test]
  21 fn test_simple_monitor_permanent_update_fail() {
  22         // Test that we handle a simple permanent monitor update failure
  23         let mut nodes = create_network(2);
  24         create_announced_chan_between_nodes(&nodes, 0, 1);
  25
  26         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  27         let (_, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
  28
  29         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  30         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  31         check_added_monitors!(nodes[0], 1);
  32
  33         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  34         assert_eq!(events_1.len(), 2);
  35         match events_1[0] {
  36                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  37                 _ => panic!("Unexpected event"),
  38         };
  39         match events_1[1] {
  40                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  41                 _ => panic!("Unexpected event"),
  42         };
  43
  44         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  45         // PaymentFailed event
  46
  47         assert_eq!(nodes[0].node.list_channels().len(), 0);
  48 }
  49
  50 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  51         // Test that we can recover from a simple temporary monitor update failure optionally with
  52         // a disconnect in between
  53         let mut nodes = create_network(2);
  54         create_announced_chan_between_nodes(&nodes, 0, 1);
  55
  56         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  57         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
  58
  59         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  60         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  61         check_added_monitors!(nodes[0], 1);
  62
  63         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  64         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  65         assert_eq!(nodes[0].node.list_channels().len(), 1);
  66
  67         if disconnect {
  68                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  69                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  70                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  71         }
  72
  73         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  74         nodes[0].node.test_restore_channel_monitor();
  75         check_added_monitors!(nodes[0], 1);
  76
  77         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  78         assert_eq!(events_2.len(), 1);
  79         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  80         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  81         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]).unwrap();
  82         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  83
  84         expect_pending_htlcs_forwardable!(nodes[1]);
  85
  86         let events_3 = nodes[1].node.get_and_clear_pending_events();
  87         assert_eq!(events_3.len(), 1);
  88         match events_3[0] {
  89                 Event::PaymentReceived { ref payment_hash, amt } => {
  90                         assert_eq!(payment_hash_1, *payment_hash);
  91                         assert_eq!(amt, 1000000);
  92                 },
  93                 _ => panic!("Unexpected event"),
  94         }
  95
  96         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
  97
  98         // Now set it to failed again...
  99         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 100         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 101         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 102         check_added_monitors!(nodes[0], 1);
 103
 104         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 105         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 106         assert_eq!(nodes[0].node.list_channels().len(), 1);
 107
 108         if disconnect {
 109                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 110                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 111                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 112         }
 113
 114         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 115         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 116         nodes[0].node.test_restore_channel_monitor();
 117         check_added_monitors!(nodes[0], 1);
 118         check_closed_broadcast!(nodes[0]);
 119
 120         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 121         // PaymentFailed event
 122
 123         assert_eq!(nodes[0].node.list_channels().len(), 0);
 124 }
 125
 126 #[test]
 127 fn test_simple_monitor_temporary_update_fail() {
 128         do_test_simple_monitor_temporary_update_fail(false);
 129         do_test_simple_monitor_temporary_update_fail(true);
 130 }
 131
 132 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 133         let disconnect_flags = 8 | 16;
 134
 135         // Test that we can recover from a temporary monitor update failure with some in-flight
 136         // HTLCs going on at the same time potentially with some disconnection thrown in.
 137         // * First we route a payment, then get a temporary monitor update failure when trying to
 138         //   route a second payment. We then claim the first payment.
 139         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 140         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 141         //   the ChannelMonitor on a watchtower).
 142         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 143         //   immediately, otherwise we wait sconnect and deliver them via the reconnect
 144         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 145         //   disconnect_count & !disconnect_flags is 0).
 146         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 147         //   through message sending, potentially disconnect/reconnecting multiple times based on
 148         //   disconnect_count, to get the update_fulfill_htlc through.
 149         // * We then walk through more message exchanges to get the original update_add_htlc
 150         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 151         //   disconnect/reconnecting based on disconnect_count.
 152         let mut nodes = create_network(2);
 153         create_announced_chan_between_nodes(&nodes, 0, 1);
 154
 155         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 156
 157         // Now try to send a second payment which will fail to send
 158         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 159         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 160
 161         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 162         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 163         check_added_monitors!(nodes[0], 1);
 164
 165         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 166         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 167         assert_eq!(nodes[0].node.list_channels().len(), 1);
 168
 169         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 170         // but nodes[0] won't respond since it is frozen.
 171         assert!(nodes[1].node.claim_funds(payment_preimage_1));
 172         check_added_monitors!(nodes[1], 1);
 173         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 174         assert_eq!(events_2.len(), 1);
 175         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 176                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 177                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 178                         assert!(update_add_htlcs.is_empty());
 179                         assert_eq!(update_fulfill_htlcs.len(), 1);
 180                         assert!(update_fail_htlcs.is_empty());
 181                         assert!(update_fail_malformed_htlcs.is_empty());
 182                         assert!(update_fee.is_none());
 183
 184                         if (disconnect_count & 16) == 0 {
 185                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]).unwrap();
 186                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 187                                 assert_eq!(events_3.len(), 1);
 188                                 match events_3[0] {
 189                                         Event::PaymentSent { ref payment_preimage } => {
 190                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 191                                         },
 192                                         _ => panic!("Unexpected event"),
 193                                 }
 194
 195                                 if let Err(msgs::HandleError{err, action: Some(msgs::ErrorAction::IgnoreError) }) = nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed) {
 196                                         assert_eq!(err, "Previous monitor update failure prevented generation of RAA");
 197                                 } else { panic!(); }
 198                         }
 199
 200                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 201                 },
 202                 _ => panic!("Unexpected event"),
 203         };
 204
 205         if disconnect_count & !disconnect_flags > 0 {
 206                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 207                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 208         }
 209
 210         // Now fix monitor updating...
 211         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 212         nodes[0].node.test_restore_channel_monitor();
 213         check_added_monitors!(nodes[0], 1);
 214
 215         macro_rules! disconnect_reconnect_peers { () => { {
 216                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 217                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 218
 219                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id());
 220                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 221                 assert_eq!(reestablish_1.len(), 1);
 222                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id());
 223                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 224                 assert_eq!(reestablish_2.len(), 1);
 225
 226                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]).unwrap();
 227                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 228                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]).unwrap();
 229                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 230
 231                 assert!(as_resp.0.is_none());
 232                 assert!(bs_resp.0.is_none());
 233
 234                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 235         } } }
 236
 237         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 238                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 239                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 240
 241                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id());
 242                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 243                 assert_eq!(reestablish_1.len(), 1);
 244                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id());
 245                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 246                 assert_eq!(reestablish_2.len(), 1);
 247
 248                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]).unwrap();
 249                 check_added_monitors!(nodes[0], 0);
 250                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 251                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]).unwrap();
 252                 check_added_monitors!(nodes[1], 0);
 253                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 254
 255                 assert!(as_resp.0.is_none());
 256                 assert!(bs_resp.0.is_none());
 257
 258                 assert!(bs_resp.1.is_none());
 259                 if (disconnect_count & 16) == 0 {
 260                         assert!(bs_resp.2.is_none());
 261
 262                         assert!(as_resp.1.is_some());
 263                         assert!(as_resp.2.is_some());
 264                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 265                 } else {
 266                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 267                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 268                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 269                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 270                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 271                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 272
 273                         assert!(as_resp.1.is_none());
 274
 275                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]).unwrap();
 276                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 277                         assert_eq!(events_3.len(), 1);
 278                         match events_3[0] {
 279                                 Event::PaymentSent { ref payment_preimage } => {
 280                                         assert_eq!(*payment_preimage, payment_preimage_1);
 281                                 },
 282                                 _ => panic!("Unexpected event"),
 283                         }
 284
 285                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed).unwrap();
 286                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 287                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 288                         check_added_monitors!(nodes[0], 1);
 289
 290                         as_resp.1 = Some(as_resp_raa);
 291                         bs_resp.2 = None;
 292                 }
 293
 294                 if disconnect_count & !disconnect_flags > 1 {
 295                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 296
 297                         if (disconnect_count & 16) == 0 {
 298                                 assert!(reestablish_1 == second_reestablish_1);
 299                                 assert!(reestablish_2 == second_reestablish_2);
 300                         }
 301                         assert!(as_resp == second_as_resp);
 302                         assert!(bs_resp == second_bs_resp);
 303                 }
 304
 305                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 306         } else {
 307                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 308                 assert_eq!(events_4.len(), 2);
 309                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 310                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 311                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 312                                 msg.clone()
 313                         },
 314                         _ => panic!("Unexpected event"),
 315                 })
 316         };
 317
 318         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 319
 320         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]).unwrap();
 321         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg).unwrap();
 322         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 323         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 324         check_added_monitors!(nodes[1], 1);
 325
 326         if disconnect_count & !disconnect_flags > 2 {
 327                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 328
 329                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 330                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 331
 332                 assert!(as_resp.2.is_none());
 333                 assert!(bs_resp.2.is_none());
 334         }
 335
 336         let as_commitment_update;
 337         let bs_second_commitment_update;
 338
 339         macro_rules! handle_bs_raa { () => {
 340                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack).unwrap();
 341                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 342                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 343                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 344                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 345                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 346                 assert!(as_commitment_update.update_fee.is_none());
 347                 check_added_monitors!(nodes[0], 1);
 348         } }
 349
 350         macro_rules! handle_initial_raa { () => {
 351                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack).unwrap();
 352                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 353                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 354                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 355                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 356                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 357                 assert!(bs_second_commitment_update.update_fee.is_none());
 358                 check_added_monitors!(nodes[1], 1);
 359         } }
 360
 361         if (disconnect_count & 8) == 0 {
 362                 handle_bs_raa!();
 363
 364                 if disconnect_count & !disconnect_flags > 3 {
 365                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 366
 367                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 368                         assert!(bs_resp.1.is_none());
 369
 370                         assert!(as_resp.2.unwrap() == as_commitment_update);
 371                         assert!(bs_resp.2.is_none());
 372
 373                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 374                 }
 375
 376                 handle_initial_raa!();
 377
 378                 if disconnect_count & !disconnect_flags > 4 {
 379                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 380
 381                         assert!(as_resp.1.is_none());
 382                         assert!(bs_resp.1.is_none());
 383
 384                         assert!(as_resp.2.unwrap() == as_commitment_update);
 385                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 386                 }
 387         } else {
 388                 handle_initial_raa!();
 389
 390                 if disconnect_count & !disconnect_flags > 3 {
 391                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 392
 393                         assert!(as_resp.1.is_none());
 394                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 395
 396                         assert!(as_resp.2.is_none());
 397                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 398
 399                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 400                 }
 401
 402                 handle_bs_raa!();
 403
 404                 if disconnect_count & !disconnect_flags > 4 {
 405                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 406
 407                         assert!(as_resp.1.is_none());
 408                         assert!(bs_resp.1.is_none());
 409
 410                         assert!(as_resp.2.unwrap() == as_commitment_update);
 411                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 412                 }
 413         }
 414
 415         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed).unwrap();
 416         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 417         // No commitment_signed so get_event_msg's assert(len == 1) passes
 418         check_added_monitors!(nodes[0], 1);
 419
 420         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed).unwrap();
 421         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 422         // No commitment_signed so get_event_msg's assert(len == 1) passes
 423         check_added_monitors!(nodes[1], 1);
 424
 425         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack).unwrap();
 426         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 427         check_added_monitors!(nodes[1], 1);
 428
 429         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack).unwrap();
 430         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 431         check_added_monitors!(nodes[0], 1);
 432
 433         expect_pending_htlcs_forwardable!(nodes[1]);
 434
 435         let events_5 = nodes[1].node.get_and_clear_pending_events();
 436         assert_eq!(events_5.len(), 1);
 437         match events_5[0] {
 438                 Event::PaymentReceived { ref payment_hash, amt } => {
 439                         assert_eq!(payment_hash_2, *payment_hash);
 440                         assert_eq!(amt, 1000000);
 441                 },
 442                 _ => panic!("Unexpected event"),
 443         }
 444
 445         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2);
 446 }
 447
 448 #[test]
 449 fn test_monitor_temporary_update_fail_a() {
 450         do_test_monitor_temporary_update_fail(0);
 451         do_test_monitor_temporary_update_fail(1);
 452         do_test_monitor_temporary_update_fail(2);
 453         do_test_monitor_temporary_update_fail(3);
 454         do_test_monitor_temporary_update_fail(4);
 455         do_test_monitor_temporary_update_fail(5);
 456 }
 457
 458 #[test]
 459 fn test_monitor_temporary_update_fail_b() {
 460         do_test_monitor_temporary_update_fail(2 | 8);
 461         do_test_monitor_temporary_update_fail(3 | 8);
 462         do_test_monitor_temporary_update_fail(4 | 8);
 463         do_test_monitor_temporary_update_fail(5 | 8);
 464 }
 465
 466 #[test]
 467 fn test_monitor_temporary_update_fail_c() {
 468         do_test_monitor_temporary_update_fail(1 | 16);
 469         do_test_monitor_temporary_update_fail(2 | 16);
 470         do_test_monitor_temporary_update_fail(3 | 16);
 471         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 472         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 473 }
 474
 475 #[test]
 476 fn test_monitor_update_fail_cs() {
 477         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 478         let mut nodes = create_network(2);
 479         create_announced_chan_between_nodes(&nodes, 0, 1);
 480
 481         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 482         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 483         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 484         check_added_monitors!(nodes[0], 1);
 485
 486         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 487         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 488
 489         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 490         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg).unwrap_err() {
 491                 assert_eq!(err, "Failed to update ChannelMonitor");
 492         } else { panic!(); }
 493         check_added_monitors!(nodes[1], 1);
 494         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 495
 496         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 497         nodes[1].node.test_restore_channel_monitor();
 498         check_added_monitors!(nodes[1], 1);
 499         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 500         assert_eq!(responses.len(), 2);
 501
 502         match responses[0] {
 503                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 504                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 505                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg).unwrap();
 506                         check_added_monitors!(nodes[0], 1);
 507                 },
 508                 _ => panic!("Unexpected event"),
 509         }
 510         match responses[1] {
 511                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 512                         assert!(updates.update_add_htlcs.is_empty());
 513                         assert!(updates.update_fulfill_htlcs.is_empty());
 514                         assert!(updates.update_fail_htlcs.is_empty());
 515                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 516                         assert!(updates.update_fee.is_none());
 517                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 518
 519                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 520                         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed).unwrap_err() {
 521                                 assert_eq!(err, "Failed to update ChannelMonitor");
 522                         } else { panic!(); }
 523                         check_added_monitors!(nodes[0], 1);
 524                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 525                 },
 526                 _ => panic!("Unexpected event"),
 527         }
 528
 529         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 530         nodes[0].node.test_restore_channel_monitor();
 531         check_added_monitors!(nodes[0], 1);
 532
 533         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 534         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa).unwrap();
 535         check_added_monitors!(nodes[1], 1);
 536
 537         expect_pending_htlcs_forwardable!(nodes[1]);
 538
 539         let events = nodes[1].node.get_and_clear_pending_events();
 540         assert_eq!(events.len(), 1);
 541         match events[0] {
 542                 Event::PaymentReceived { payment_hash, amt } => {
 543                         assert_eq!(payment_hash, our_payment_hash);
 544                         assert_eq!(amt, 1000000);
 545                 },
 546                 _ => panic!("Unexpected event"),
 547         };
 548
 549         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 550 }
 551
 552 #[test]
 553 fn test_monitor_update_fail_no_rebroadcast() {
 554         // Tests handling of a monitor update failure when no message rebroadcasting on
 555         // test_restore_channel_monitor() is required. Backported from
 556         // chanmon_fail_consistency fuzz tests.
 557         let mut nodes = create_network(2);
 558         create_announced_chan_between_nodes(&nodes, 0, 1);
 559
 560         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 561         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 562         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 563         check_added_monitors!(nodes[0], 1);
 564
 565         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 566         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 567         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 568
 569         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 570         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa).unwrap_err() {
 571                 assert_eq!(err, "Failed to update ChannelMonitor");
 572         } else { panic!(); }
 573         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 574         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 575         check_added_monitors!(nodes[1], 1);
 576
 577         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 578         nodes[1].node.test_restore_channel_monitor();
 579         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 580         check_added_monitors!(nodes[1], 1);
 581         expect_pending_htlcs_forwardable!(nodes[1]);
 582
 583         let events = nodes[1].node.get_and_clear_pending_events();
 584         assert_eq!(events.len(), 1);
 585         match events[0] {
 586                 Event::PaymentReceived { payment_hash, .. } => {
 587                         assert_eq!(payment_hash, our_payment_hash);
 588                 },
 589                 _ => panic!("Unexpected event"),
 590         }
 591
 592         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 593 }
 594
 595 #[test]
 596 fn test_monitor_update_raa_while_paused() {
 597         // Tests handling of an RAA while monitor updating has already been marked failed.
 598         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 599         let mut nodes = create_network(2);
 600         create_announced_chan_between_nodes(&nodes, 0, 1);
 601
 602         send_payment(&nodes[0], &[&nodes[1]], 5000000);
 603
 604         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 605         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 606         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 607         check_added_monitors!(nodes[0], 1);
 608         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 609
 610         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 611         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 612         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 613         check_added_monitors!(nodes[1], 1);
 614         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 615
 616         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]).unwrap();
 617         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg).unwrap();
 618         check_added_monitors!(nodes[1], 1);
 619         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 620
 621         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 622         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]).unwrap();
 623         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg).unwrap_err() {
 624                 assert_eq!(err, "Failed to update ChannelMonitor");
 625         } else { panic!(); }
 626         check_added_monitors!(nodes[0], 1);
 627
 628         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa).unwrap_err() {
 629                 assert_eq!(err, "Previous monitor update failure prevented responses to RAA");
 630         } else { panic!(); }
 631         check_added_monitors!(nodes[0], 1);
 632
 633         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 634         nodes[0].node.test_restore_channel_monitor();
 635         check_added_monitors!(nodes[0], 1);
 636
 637         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 638         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0).unwrap();
 639         check_added_monitors!(nodes[1], 1);
 640         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 641
 642         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1).unwrap();
 643         check_added_monitors!(nodes[1], 1);
 644         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 645
 646         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed).unwrap();
 647         check_added_monitors!(nodes[0], 1);
 648         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 649
 650         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa).unwrap();
 651         check_added_monitors!(nodes[0], 1);
 652         expect_pending_htlcs_forwardable!(nodes[0]);
 653         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 654
 655         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa).unwrap();
 656         check_added_monitors!(nodes[1], 1);
 657         expect_pending_htlcs_forwardable!(nodes[1]);
 658         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 659
 660         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1);
 661         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2);
 662 }
 663
 664 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 665         // Tests handling of a monitor update failure when processing an incoming RAA
 666         let mut nodes = create_network(3);
 667         create_announced_chan_between_nodes(&nodes, 0, 1);
 668         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2);
 669
 670         // Rebalance a bit so that we can send backwards from 2 to 1.
 671         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000);
 672
 673         // Route a first payment that we'll fail backwards
 674         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 675
 676         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 677         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, 0));
 678         expect_pending_htlcs_forwardable!(nodes[2]);
 679         check_added_monitors!(nodes[2], 1);
 680
 681         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 682         assert!(updates.update_add_htlcs.is_empty());
 683         assert!(updates.update_fulfill_htlcs.is_empty());
 684         assert_eq!(updates.update_fail_htlcs.len(), 1);
 685         assert!(updates.update_fail_malformed_htlcs.is_empty());
 686         assert!(updates.update_fee.is_none());
 687         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]).unwrap();
 688
 689         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 690         check_added_monitors!(nodes[0], 0);
 691
 692         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 693         // holding cell.
 694         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 695         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 696         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 697         check_added_monitors!(nodes[0], 1);
 698
 699         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 700         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 701         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 702
 703         expect_pending_htlcs_forwardable!(nodes[1]);
 704         check_added_monitors!(nodes[1], 0);
 705         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 706
 707         // Now fail monitor updating.
 708         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 709         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack).unwrap_err() {
 710                 assert_eq!(err, "Failed to update ChannelMonitor");
 711         } else { panic!(); }
 712         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 713         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 714         check_added_monitors!(nodes[1], 1);
 715
 716         // Attempt to forward a third payment but fail due to the second channel being unavailable
 717         // for forwarding.
 718
 719         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 720         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 721         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 722         check_added_monitors!(nodes[0], 1);
 723
 724         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 725         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 726         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 727         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 728         check_added_monitors!(nodes[1], 0);
 729
 730         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 731         assert_eq!(events_2.len(), 1);
 732         match events_2.remove(0) {
 733                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 734                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 735                         assert!(updates.update_fulfill_htlcs.is_empty());
 736                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 737                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 738                         assert!(updates.update_add_htlcs.is_empty());
 739                         assert!(updates.update_fee.is_none());
 740
 741                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]).unwrap();
 742                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 743
 744                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 745                         assert_eq!(msg_events.len(), 1);
 746                         match msg_events[0] {
 747                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 748                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 749                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 750                                 },
 751                                 _ => panic!("Unexpected event"),
 752                         }
 753
 754                         let events = nodes[0].node.get_and_clear_pending_events();
 755                         assert_eq!(events.len(), 1);
 756                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 757                                 assert_eq!(payment_hash, payment_hash_3);
 758                                 assert!(!rejected_by_dest);
 759                         } else { panic!("Unexpected event!"); }
 760                 },
 761                 _ => panic!("Unexpected event type!"),
 762         };
 763
 764         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 765                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 766                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 767                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 768                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 769                 check_added_monitors!(nodes[2], 1);
 770
 771                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 772                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 773                 if let Err(msgs::HandleError{err, action: Some(msgs::ErrorAction::IgnoreError) }) = nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg) {
 774                         assert_eq!(err, "Previous monitor update failure prevented generation of RAA");
 775                 } else { panic!(); }
 776                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 777                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 778                 (Some(payment_preimage_4), Some(payment_hash_4))
 779         } else { (None, None) };
 780
 781         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 782         // update_add update.
 783         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 784         nodes[1].node.test_restore_channel_monitor();
 785         check_added_monitors!(nodes[1], 1);
 786         expect_pending_htlcs_forwardable!(nodes[1]);
 787         check_added_monitors!(nodes[1], 1);
 788
 789         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 790         if test_ignore_second_cs {
 791                 assert_eq!(events_3.len(), 3);
 792         } else {
 793                 assert_eq!(events_3.len(), 2);
 794         }
 795
 796         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 797         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 798         let messages_a = match events_3.pop().unwrap() {
 799                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 800                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 801                         assert!(updates.update_fulfill_htlcs.is_empty());
 802                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 803                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 804                         assert!(updates.update_add_htlcs.is_empty());
 805                         assert!(updates.update_fee.is_none());
 806                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 807                 },
 808                 _ => panic!("Unexpected event type!"),
 809         };
 810         let raa = if test_ignore_second_cs {
 811                 match events_3.remove(1) {
 812                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 813                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 814                                 Some(msg.clone())
 815                         },
 816                         _ => panic!("Unexpected event"),
 817                 }
 818         } else { None };
 819         let send_event_b = SendEvent::from_event(events_3.remove(0));
 820         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 821
 822         // Now deliver the new messages...
 823
 824         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0).unwrap();
 825         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 826         let events_4 = nodes[0].node.get_and_clear_pending_events();
 827         assert_eq!(events_4.len(), 1);
 828         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 829                 assert_eq!(payment_hash, payment_hash_1);
 830                 assert!(rejected_by_dest);
 831         } else { panic!("Unexpected event!"); }
 832
 833         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]).unwrap();
 834         if test_ignore_second_cs {
 835                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg).unwrap();
 836                 check_added_monitors!(nodes[2], 1);
 837                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 838                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap()).unwrap();
 839                 check_added_monitors!(nodes[2], 1);
 840                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 841                 assert!(bs_cs.update_add_htlcs.is_empty());
 842                 assert!(bs_cs.update_fail_htlcs.is_empty());
 843                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 844                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 845                 assert!(bs_cs.update_fee.is_none());
 846
 847                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack).unwrap();
 848                 check_added_monitors!(nodes[1], 1);
 849                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 850                 assert!(as_cs.update_add_htlcs.is_empty());
 851                 assert!(as_cs.update_fail_htlcs.is_empty());
 852                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 853                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 854                 assert!(as_cs.update_fee.is_none());
 855
 856                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed).unwrap();
 857                 check_added_monitors!(nodes[1], 1);
 858                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 859
 860                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed).unwrap();
 861                 check_added_monitors!(nodes[2], 1);
 862                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 863
 864                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa).unwrap();
 865                 check_added_monitors!(nodes[2], 1);
 866                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 867
 868                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa).unwrap();
 869                 check_added_monitors!(nodes[1], 1);
 870                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 871         } else {
 872                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 873         }
 874
 875         expect_pending_htlcs_forwardable!(nodes[2]);
 876
 877         let events_6 = nodes[2].node.get_and_clear_pending_events();
 878         assert_eq!(events_6.len(), 1);
 879         match events_6[0] {
 880                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 881                 _ => panic!("Unexpected event"),
 882         };
 883
 884         if test_ignore_second_cs {
 885                 expect_pending_htlcs_forwardable!(nodes[1]);
 886                 check_added_monitors!(nodes[1], 1);
 887
 888                 send_event = SendEvent::from_node(&nodes[1]);
 889                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 890                 assert_eq!(send_event.msgs.len(), 1);
 891                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]).unwrap();
 892                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 893
 894                 expect_pending_htlcs_forwardable!(nodes[0]);
 895
 896                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 897                 assert_eq!(events_9.len(), 1);
 898                 match events_9[0] {
 899                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 900                         _ => panic!("Unexpected event"),
 901                 };
 902                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap());
 903         }
 904
 905         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
 906 }
 907
 908 #[test]
 909 fn test_monitor_update_fail_raa() {
 910         do_test_monitor_update_fail_raa(false);
 911         do_test_monitor_update_fail_raa(true);
 912 }
 913
 914 #[test]
 915 fn test_monitor_update_fail_reestablish() {
 916         // Simple test for message retransmission after monitor update failure on
 917         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 918         // HTLCs).
 919         let mut nodes = create_network(3);
 920         create_announced_chan_between_nodes(&nodes, 0, 1);
 921         create_announced_chan_between_nodes(&nodes, 1, 2);
 922
 923         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 924
 925         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 926         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 927
 928         assert!(nodes[2].node.claim_funds(our_payment_preimage));
 929         check_added_monitors!(nodes[2], 1);
 930         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 931         assert!(updates.update_add_htlcs.is_empty());
 932         assert!(updates.update_fail_htlcs.is_empty());
 933         assert!(updates.update_fail_malformed_htlcs.is_empty());
 934         assert!(updates.update_fee.is_none());
 935         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 936         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]).unwrap();
 937         check_added_monitors!(nodes[1], 1);
 938         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 939         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 940
 941         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 942         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id());
 943         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id());
 944
 945         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 946         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 947
 948         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish).unwrap();
 949
 950         if let msgs::HandleError { err, action: Some(msgs::ErrorAction::IgnoreError) } = nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish).unwrap_err() {
 951                 assert_eq!(err, "Failed to update ChannelMonitor");
 952         } else { panic!(); }
 953         check_added_monitors!(nodes[1], 1);
 954
 955         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 956         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 957
 958         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id());
 959         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id());
 960
 961         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 962         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 963
 964         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish).unwrap();
 965
 966         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish).unwrap();
 967         check_added_monitors!(nodes[1], 0);
 968         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 969
 970         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 971         nodes[1].node.test_restore_channel_monitor();
 972         check_added_monitors!(nodes[1], 1);
 973
 974         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 975         assert!(updates.update_add_htlcs.is_empty());
 976         assert!(updates.update_fail_htlcs.is_empty());
 977         assert!(updates.update_fail_malformed_htlcs.is_empty());
 978         assert!(updates.update_fee.is_none());
 979         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 980         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]).unwrap();
 981         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
 982
 983         let events = nodes[0].node.get_and_clear_pending_events();
 984         assert_eq!(events.len(), 1);
 985         match events[0] {
 986                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
 987                 _ => panic!("Unexpected event"),
 988         }
 989 }