_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let node_cfgs = create_node_cfgs(2);
  23         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  24         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  25         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  26
  27         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  28         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  29
  30         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  31         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  32         check_added_monitors!(nodes[0], 1);
  33
  34         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  35         assert_eq!(events_1.len(), 2);
  36         match events_1[0] {
  37                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  38                 _ => panic!("Unexpected event"),
  39         };
  40         match events_1[1] {
  41                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  42                 _ => panic!("Unexpected event"),
  43         };
  44
  45         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  46         // PaymentFailed event
  47
  48         assert_eq!(nodes[0].node.list_channels().len(), 0);
  49 }
  50
  51 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  52         // Test that we can recover from a simple temporary monitor update failure optionally with
  53         // a disconnect in between
  54         let node_cfgs = create_node_cfgs(2);
  55         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  56         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  57         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  58
  59         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  60         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  61
  62         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  63         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  64         check_added_monitors!(nodes[0], 1);
  65
  66         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  67         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  68         assert_eq!(nodes[0].node.list_channels().len(), 1);
  69
  70         if disconnect {
  71                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  72                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  73                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  74         }
  75
  76         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  77         nodes[0].node.test_restore_channel_monitor();
  78         check_added_monitors!(nodes[0], 1);
  79
  80         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  81         assert_eq!(events_2.len(), 1);
  82         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  83         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  84         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  85         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  86
  87         expect_pending_htlcs_forwardable!(nodes[1]);
  88
  89         let events_3 = nodes[1].node.get_and_clear_pending_events();
  90         assert_eq!(events_3.len(), 1);
  91         match events_3[0] {
  92                 Event::PaymentReceived { ref payment_hash, amt } => {
  93                         assert_eq!(payment_hash_1, *payment_hash);
  94                         assert_eq!(amt, 1000000);
  95                 },
  96                 _ => panic!("Unexpected event"),
  97         }
  98
  99         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 100
 101         // Now set it to failed again...
 102         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 103         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 104         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 105         check_added_monitors!(nodes[0], 1);
 106
 107         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 108         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 109         assert_eq!(nodes[0].node.list_channels().len(), 1);
 110
 111         if disconnect {
 112                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 113                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 114                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 115         }
 116
 117         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 118         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 119         nodes[0].node.test_restore_channel_monitor();
 120         check_added_monitors!(nodes[0], 1);
 121         check_closed_broadcast!(nodes[0], false);
 122
 123         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 124         // PaymentFailed event
 125
 126         assert_eq!(nodes[0].node.list_channels().len(), 0);
 127 }
 128
 129 #[test]
 130 fn test_simple_monitor_temporary_update_fail() {
 131         do_test_simple_monitor_temporary_update_fail(false);
 132         do_test_simple_monitor_temporary_update_fail(true);
 133 }
 134
 135 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 136         let disconnect_flags = 8 | 16;
 137
 138         // Test that we can recover from a temporary monitor update failure with some in-flight
 139         // HTLCs going on at the same time potentially with some disconnection thrown in.
 140         // * First we route a payment, then get a temporary monitor update failure when trying to
 141         //   route a second payment. We then claim the first payment.
 142         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 143         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 144         //   the ChannelMonitor on a watchtower).
 145         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 146         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 147         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 148         //   disconnect_count & !disconnect_flags is 0).
 149         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 150         //   through message sending, potentially disconnect/reconnecting multiple times based on
 151         //   disconnect_count, to get the update_fulfill_htlc through.
 152         // * We then walk through more message exchanges to get the original update_add_htlc
 153         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 154         //   disconnect/reconnecting based on disconnect_count.
 155         let node_cfgs = create_node_cfgs(2);
 156         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 157         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 158         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 159
 160         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 161
 162         // Now try to send a second payment which will fail to send
 163         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 164         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 165
 166         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 167         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 168         check_added_monitors!(nodes[0], 1);
 169
 170         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 171         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 172         assert_eq!(nodes[0].node.list_channels().len(), 1);
 173
 174         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 175         // but nodes[0] won't respond since it is frozen.
 176         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
 177         check_added_monitors!(nodes[1], 1);
 178         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 179         assert_eq!(events_2.len(), 1);
 180         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 181                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 182                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 183                         assert!(update_add_htlcs.is_empty());
 184                         assert_eq!(update_fulfill_htlcs.len(), 1);
 185                         assert!(update_fail_htlcs.is_empty());
 186                         assert!(update_fail_malformed_htlcs.is_empty());
 187                         assert!(update_fee.is_none());
 188
 189                         if (disconnect_count & 16) == 0 {
 190                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 191                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 192                                 assert_eq!(events_3.len(), 1);
 193                                 match events_3[0] {
 194                                         Event::PaymentSent { ref payment_preimage } => {
 195                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 196                                         },
 197                                         _ => panic!("Unexpected event"),
 198                                 }
 199
 200                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 201                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 202                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 203                         }
 204
 205                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 206                 },
 207                 _ => panic!("Unexpected event"),
 208         };
 209
 210         if disconnect_count & !disconnect_flags > 0 {
 211                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 212                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 213         }
 214
 215         // Now fix monitor updating...
 216         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 217         nodes[0].node.test_restore_channel_monitor();
 218         check_added_monitors!(nodes[0], 1);
 219
 220         macro_rules! disconnect_reconnect_peers { () => { {
 221                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 222                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 223
 224                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 225                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 226                 assert_eq!(reestablish_1.len(), 1);
 227                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 228                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 229                 assert_eq!(reestablish_2.len(), 1);
 230
 231                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 232                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 233                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 234                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 235
 236                 assert!(as_resp.0.is_none());
 237                 assert!(bs_resp.0.is_none());
 238
 239                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 240         } } }
 241
 242         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 243                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 244                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 245
 246                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 247                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 248                 assert_eq!(reestablish_1.len(), 1);
 249                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 250                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 251                 assert_eq!(reestablish_2.len(), 1);
 252
 253                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 254                 check_added_monitors!(nodes[0], 0);
 255                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 256                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 257                 check_added_monitors!(nodes[1], 0);
 258                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 259
 260                 assert!(as_resp.0.is_none());
 261                 assert!(bs_resp.0.is_none());
 262
 263                 assert!(bs_resp.1.is_none());
 264                 if (disconnect_count & 16) == 0 {
 265                         assert!(bs_resp.2.is_none());
 266
 267                         assert!(as_resp.1.is_some());
 268                         assert!(as_resp.2.is_some());
 269                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 270                 } else {
 271                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 272                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 273                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 274                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 275                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 276                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 277
 278                         assert!(as_resp.1.is_none());
 279
 280                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 281                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 282                         assert_eq!(events_3.len(), 1);
 283                         match events_3[0] {
 284                                 Event::PaymentSent { ref payment_preimage } => {
 285                                         assert_eq!(*payment_preimage, payment_preimage_1);
 286                                 },
 287                                 _ => panic!("Unexpected event"),
 288                         }
 289
 290                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 291                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 292                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 293                         check_added_monitors!(nodes[0], 1);
 294
 295                         as_resp.1 = Some(as_resp_raa);
 296                         bs_resp.2 = None;
 297                 }
 298
 299                 if disconnect_count & !disconnect_flags > 1 {
 300                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 301
 302                         if (disconnect_count & 16) == 0 {
 303                                 assert!(reestablish_1 == second_reestablish_1);
 304                                 assert!(reestablish_2 == second_reestablish_2);
 305                         }
 306                         assert!(as_resp == second_as_resp);
 307                         assert!(bs_resp == second_bs_resp);
 308                 }
 309
 310                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 311         } else {
 312                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 313                 assert_eq!(events_4.len(), 2);
 314                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 315                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 316                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 317                                 msg.clone()
 318                         },
 319                         _ => panic!("Unexpected event"),
 320                 })
 321         };
 322
 323         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 324
 325         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 326         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 327         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 328         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 329         check_added_monitors!(nodes[1], 1);
 330
 331         if disconnect_count & !disconnect_flags > 2 {
 332                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 333
 334                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 335                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 336
 337                 assert!(as_resp.2.is_none());
 338                 assert!(bs_resp.2.is_none());
 339         }
 340
 341         let as_commitment_update;
 342         let bs_second_commitment_update;
 343
 344         macro_rules! handle_bs_raa { () => {
 345                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 346                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 347                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 348                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 349                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 350                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 351                 assert!(as_commitment_update.update_fee.is_none());
 352                 check_added_monitors!(nodes[0], 1);
 353         } }
 354
 355         macro_rules! handle_initial_raa { () => {
 356                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 357                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 358                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 359                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 360                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 361                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 362                 assert!(bs_second_commitment_update.update_fee.is_none());
 363                 check_added_monitors!(nodes[1], 1);
 364         } }
 365
 366         if (disconnect_count & 8) == 0 {
 367                 handle_bs_raa!();
 368
 369                 if disconnect_count & !disconnect_flags > 3 {
 370                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 371
 372                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 373                         assert!(bs_resp.1.is_none());
 374
 375                         assert!(as_resp.2.unwrap() == as_commitment_update);
 376                         assert!(bs_resp.2.is_none());
 377
 378                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 379                 }
 380
 381                 handle_initial_raa!();
 382
 383                 if disconnect_count & !disconnect_flags > 4 {
 384                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 385
 386                         assert!(as_resp.1.is_none());
 387                         assert!(bs_resp.1.is_none());
 388
 389                         assert!(as_resp.2.unwrap() == as_commitment_update);
 390                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 391                 }
 392         } else {
 393                 handle_initial_raa!();
 394
 395                 if disconnect_count & !disconnect_flags > 3 {
 396                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 397
 398                         assert!(as_resp.1.is_none());
 399                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 400
 401                         assert!(as_resp.2.is_none());
 402                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 403
 404                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 405                 }
 406
 407                 handle_bs_raa!();
 408
 409                 if disconnect_count & !disconnect_flags > 4 {
 410                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 411
 412                         assert!(as_resp.1.is_none());
 413                         assert!(bs_resp.1.is_none());
 414
 415                         assert!(as_resp.2.unwrap() == as_commitment_update);
 416                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 417                 }
 418         }
 419
 420         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 421         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 422         // No commitment_signed so get_event_msg's assert(len == 1) passes
 423         check_added_monitors!(nodes[0], 1);
 424
 425         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 426         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 427         // No commitment_signed so get_event_msg's assert(len == 1) passes
 428         check_added_monitors!(nodes[1], 1);
 429
 430         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 431         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 432         check_added_monitors!(nodes[1], 1);
 433
 434         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 435         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 436         check_added_monitors!(nodes[0], 1);
 437
 438         expect_pending_htlcs_forwardable!(nodes[1]);
 439
 440         let events_5 = nodes[1].node.get_and_clear_pending_events();
 441         assert_eq!(events_5.len(), 1);
 442         match events_5[0] {
 443                 Event::PaymentReceived { ref payment_hash, amt } => {
 444                         assert_eq!(payment_hash_2, *payment_hash);
 445                         assert_eq!(amt, 1000000);
 446                 },
 447                 _ => panic!("Unexpected event"),
 448         }
 449
 450         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 451 }
 452
 453 #[test]
 454 fn test_monitor_temporary_update_fail_a() {
 455         do_test_monitor_temporary_update_fail(0);
 456         do_test_monitor_temporary_update_fail(1);
 457         do_test_monitor_temporary_update_fail(2);
 458         do_test_monitor_temporary_update_fail(3);
 459         do_test_monitor_temporary_update_fail(4);
 460         do_test_monitor_temporary_update_fail(5);
 461 }
 462
 463 #[test]
 464 fn test_monitor_temporary_update_fail_b() {
 465         do_test_monitor_temporary_update_fail(2 | 8);
 466         do_test_monitor_temporary_update_fail(3 | 8);
 467         do_test_monitor_temporary_update_fail(4 | 8);
 468         do_test_monitor_temporary_update_fail(5 | 8);
 469 }
 470
 471 #[test]
 472 fn test_monitor_temporary_update_fail_c() {
 473         do_test_monitor_temporary_update_fail(1 | 16);
 474         do_test_monitor_temporary_update_fail(2 | 16);
 475         do_test_monitor_temporary_update_fail(3 | 16);
 476         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 477         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 478 }
 479
 480 #[test]
 481 fn test_monitor_update_fail_cs() {
 482         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 483         let node_cfgs = create_node_cfgs(2);
 484         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 485         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 486         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 487
 488         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 489         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 490         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 491         check_added_monitors!(nodes[0], 1);
 492
 493         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 494         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 495
 496         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 497         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 498         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 499         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 500         check_added_monitors!(nodes[1], 1);
 501         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 502
 503         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 504         nodes[1].node.test_restore_channel_monitor();
 505         check_added_monitors!(nodes[1], 1);
 506         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 507         assert_eq!(responses.len(), 2);
 508
 509         match responses[0] {
 510                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 511                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 512                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 513                         check_added_monitors!(nodes[0], 1);
 514                 },
 515                 _ => panic!("Unexpected event"),
 516         }
 517         match responses[1] {
 518                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 519                         assert!(updates.update_add_htlcs.is_empty());
 520                         assert!(updates.update_fulfill_htlcs.is_empty());
 521                         assert!(updates.update_fail_htlcs.is_empty());
 522                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 523                         assert!(updates.update_fee.is_none());
 524                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 525
 526                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 527                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 528                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 529                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 530                         check_added_monitors!(nodes[0], 1);
 531                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 532                 },
 533                 _ => panic!("Unexpected event"),
 534         }
 535
 536         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 537         nodes[0].node.test_restore_channel_monitor();
 538         check_added_monitors!(nodes[0], 1);
 539
 540         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 541         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 542         check_added_monitors!(nodes[1], 1);
 543
 544         expect_pending_htlcs_forwardable!(nodes[1]);
 545
 546         let events = nodes[1].node.get_and_clear_pending_events();
 547         assert_eq!(events.len(), 1);
 548         match events[0] {
 549                 Event::PaymentReceived { payment_hash, amt } => {
 550                         assert_eq!(payment_hash, our_payment_hash);
 551                         assert_eq!(amt, 1000000);
 552                 },
 553                 _ => panic!("Unexpected event"),
 554         };
 555
 556         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 557 }
 558
 559 #[test]
 560 fn test_monitor_update_fail_no_rebroadcast() {
 561         // Tests handling of a monitor update failure when no message rebroadcasting on
 562         // test_restore_channel_monitor() is required. Backported from
 563         // chanmon_fail_consistency fuzz tests.
 564         let node_cfgs = create_node_cfgs(2);
 565         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 566         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 567         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 568
 569         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 570         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 571         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 572         check_added_monitors!(nodes[0], 1);
 573
 574         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 575         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 576         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 577
 578         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 579         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 580         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 581         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 582         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 583         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 584         check_added_monitors!(nodes[1], 1);
 585
 586         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 587         nodes[1].node.test_restore_channel_monitor();
 588         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 589         check_added_monitors!(nodes[1], 1);
 590         expect_pending_htlcs_forwardable!(nodes[1]);
 591
 592         let events = nodes[1].node.get_and_clear_pending_events();
 593         assert_eq!(events.len(), 1);
 594         match events[0] {
 595                 Event::PaymentReceived { payment_hash, .. } => {
 596                         assert_eq!(payment_hash, our_payment_hash);
 597                 },
 598                 _ => panic!("Unexpected event"),
 599         }
 600
 601         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 602 }
 603
 604 #[test]
 605 fn test_monitor_update_raa_while_paused() {
 606         // Tests handling of an RAA while monitor updating has already been marked failed.
 607         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 608         let node_cfgs = create_node_cfgs(2);
 609         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 610         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 611         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 612
 613         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 614
 615         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 616         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 617         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 618         check_added_monitors!(nodes[0], 1);
 619         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 620
 621         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 622         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 623         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 624         check_added_monitors!(nodes[1], 1);
 625         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 626
 627         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 628         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 629         check_added_monitors!(nodes[1], 1);
 630         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 631
 632         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 633         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 634         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 635         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 636         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 637         check_added_monitors!(nodes[0], 1);
 638
 639         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 640         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 641         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 642         check_added_monitors!(nodes[0], 1);
 643
 644         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 645         nodes[0].node.test_restore_channel_monitor();
 646         check_added_monitors!(nodes[0], 1);
 647
 648         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 649         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 650         check_added_monitors!(nodes[1], 1);
 651         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 652
 653         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 654         check_added_monitors!(nodes[1], 1);
 655         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 656
 657         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 658         check_added_monitors!(nodes[0], 1);
 659         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 660
 661         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 662         check_added_monitors!(nodes[0], 1);
 663         expect_pending_htlcs_forwardable!(nodes[0]);
 664         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 665
 666         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 667         check_added_monitors!(nodes[1], 1);
 668         expect_pending_htlcs_forwardable!(nodes[1]);
 669         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 670
 671         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 672         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 673 }
 674
 675 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 676         // Tests handling of a monitor update failure when processing an incoming RAA
 677         let node_cfgs = create_node_cfgs(3);
 678         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 679         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 680         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 681         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 682
 683         // Rebalance a bit so that we can send backwards from 2 to 1.
 684         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 685
 686         // Route a first payment that we'll fail backwards
 687         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 688
 689         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 690         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 691         expect_pending_htlcs_forwardable!(nodes[2]);
 692         check_added_monitors!(nodes[2], 1);
 693
 694         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 695         assert!(updates.update_add_htlcs.is_empty());
 696         assert!(updates.update_fulfill_htlcs.is_empty());
 697         assert_eq!(updates.update_fail_htlcs.len(), 1);
 698         assert!(updates.update_fail_malformed_htlcs.is_empty());
 699         assert!(updates.update_fee.is_none());
 700         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 701
 702         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 703         check_added_monitors!(nodes[0], 0);
 704
 705         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 706         // holding cell.
 707         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 708         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 709         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 710         check_added_monitors!(nodes[0], 1);
 711
 712         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 713         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 714         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 715
 716         expect_pending_htlcs_forwardable!(nodes[1]);
 717         check_added_monitors!(nodes[1], 0);
 718         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 719
 720         // Now fail monitor updating.
 721         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 722         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 723         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 724         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 725         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 726         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 727         check_added_monitors!(nodes[1], 1);
 728
 729         // Attempt to forward a third payment but fail due to the second channel being unavailable
 730         // for forwarding.
 731
 732         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 733         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 734         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 735         check_added_monitors!(nodes[0], 1);
 736
 737         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 738         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 739         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 740         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 741         check_added_monitors!(nodes[1], 0);
 742
 743         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 744         assert_eq!(events_2.len(), 1);
 745         match events_2.remove(0) {
 746                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 747                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 748                         assert!(updates.update_fulfill_htlcs.is_empty());
 749                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 750                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 751                         assert!(updates.update_add_htlcs.is_empty());
 752                         assert!(updates.update_fee.is_none());
 753
 754                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 755                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 756
 757                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 758                         assert_eq!(msg_events.len(), 1);
 759                         match msg_events[0] {
 760                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 761                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 762                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 763                                 },
 764                                 _ => panic!("Unexpected event"),
 765                         }
 766
 767                         let events = nodes[0].node.get_and_clear_pending_events();
 768                         assert_eq!(events.len(), 1);
 769                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 770                                 assert_eq!(payment_hash, payment_hash_3);
 771                                 assert!(!rejected_by_dest);
 772                         } else { panic!("Unexpected event!"); }
 773                 },
 774                 _ => panic!("Unexpected event type!"),
 775         };
 776
 777         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 778                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 779                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 780                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 781                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 782                 check_added_monitors!(nodes[2], 1);
 783
 784                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 785                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 786                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 787                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 788                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 789                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 790                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 791                 (Some(payment_preimage_4), Some(payment_hash_4))
 792         } else { (None, None) };
 793
 794         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 795         // update_add update.
 796         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 797         nodes[1].node.test_restore_channel_monitor();
 798         check_added_monitors!(nodes[1], 1);
 799         expect_pending_htlcs_forwardable!(nodes[1]);
 800         check_added_monitors!(nodes[1], 1);
 801
 802         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 803         if test_ignore_second_cs {
 804                 assert_eq!(events_3.len(), 3);
 805         } else {
 806                 assert_eq!(events_3.len(), 2);
 807         }
 808
 809         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 810         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 811         let messages_a = match events_3.pop().unwrap() {
 812                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 813                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 814                         assert!(updates.update_fulfill_htlcs.is_empty());
 815                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 816                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 817                         assert!(updates.update_add_htlcs.is_empty());
 818                         assert!(updates.update_fee.is_none());
 819                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 820                 },
 821                 _ => panic!("Unexpected event type!"),
 822         };
 823         let raa = if test_ignore_second_cs {
 824                 match events_3.remove(1) {
 825                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 826                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 827                                 Some(msg.clone())
 828                         },
 829                         _ => panic!("Unexpected event"),
 830                 }
 831         } else { None };
 832         let send_event_b = SendEvent::from_event(events_3.remove(0));
 833         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 834
 835         // Now deliver the new messages...
 836
 837         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 838         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 839         let events_4 = nodes[0].node.get_and_clear_pending_events();
 840         assert_eq!(events_4.len(), 1);
 841         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 842                 assert_eq!(payment_hash, payment_hash_1);
 843                 assert!(rejected_by_dest);
 844         } else { panic!("Unexpected event!"); }
 845
 846         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 847         if test_ignore_second_cs {
 848                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 849                 check_added_monitors!(nodes[2], 1);
 850                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 851                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 852                 check_added_monitors!(nodes[2], 1);
 853                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 854                 assert!(bs_cs.update_add_htlcs.is_empty());
 855                 assert!(bs_cs.update_fail_htlcs.is_empty());
 856                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 857                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 858                 assert!(bs_cs.update_fee.is_none());
 859
 860                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 861                 check_added_monitors!(nodes[1], 1);
 862                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 863                 assert!(as_cs.update_add_htlcs.is_empty());
 864                 assert!(as_cs.update_fail_htlcs.is_empty());
 865                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 866                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 867                 assert!(as_cs.update_fee.is_none());
 868
 869                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 870                 check_added_monitors!(nodes[1], 1);
 871                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 872
 873                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 874                 check_added_monitors!(nodes[2], 1);
 875                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 876
 877                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 878                 check_added_monitors!(nodes[2], 1);
 879                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 880
 881                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 882                 check_added_monitors!(nodes[1], 1);
 883                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 884         } else {
 885                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 886         }
 887
 888         expect_pending_htlcs_forwardable!(nodes[2]);
 889
 890         let events_6 = nodes[2].node.get_and_clear_pending_events();
 891         assert_eq!(events_6.len(), 1);
 892         match events_6[0] {
 893                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 894                 _ => panic!("Unexpected event"),
 895         };
 896
 897         if test_ignore_second_cs {
 898                 expect_pending_htlcs_forwardable!(nodes[1]);
 899                 check_added_monitors!(nodes[1], 1);
 900
 901                 send_event = SendEvent::from_node(&nodes[1]);
 902                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 903                 assert_eq!(send_event.msgs.len(), 1);
 904                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 905                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 906
 907                 expect_pending_htlcs_forwardable!(nodes[0]);
 908
 909                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 910                 assert_eq!(events_9.len(), 1);
 911                 match events_9[0] {
 912                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 913                         _ => panic!("Unexpected event"),
 914                 };
 915                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 916         }
 917
 918         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 919 }
 920
 921 #[test]
 922 fn test_monitor_update_fail_raa() {
 923         do_test_monitor_update_fail_raa(false);
 924         do_test_monitor_update_fail_raa(true);
 925 }
 926
 927 #[test]
 928 fn test_monitor_update_fail_reestablish() {
 929         // Simple test for message retransmission after monitor update failure on
 930         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 931         // HTLCs).
 932         let node_cfgs = create_node_cfgs(3);
 933         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 934         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 935         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 936         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 937
 938         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 939
 940         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 941         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 942
 943         assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
 944         check_added_monitors!(nodes[2], 1);
 945         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 946         assert!(updates.update_add_htlcs.is_empty());
 947         assert!(updates.update_fail_htlcs.is_empty());
 948         assert!(updates.update_fail_malformed_htlcs.is_empty());
 949         assert!(updates.update_fee.is_none());
 950         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 951         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 952         check_added_monitors!(nodes[1], 1);
 953         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 954         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 955
 956         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 957         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 958         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 959
 960         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 961         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 962
 963         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 964
 965         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 966         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 967         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 968         check_added_monitors!(nodes[1], 1);
 969
 970         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 971         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 972
 973         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 974         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 975
 976         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 977         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 978
 979         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 980
 981         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 982         check_added_monitors!(nodes[1], 0);
 983         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 984
 985         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 986         nodes[1].node.test_restore_channel_monitor();
 987         check_added_monitors!(nodes[1], 1);
 988
 989         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 990         assert!(updates.update_add_htlcs.is_empty());
 991         assert!(updates.update_fail_htlcs.is_empty());
 992         assert!(updates.update_fail_malformed_htlcs.is_empty());
 993         assert!(updates.update_fee.is_none());
 994         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 995         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 996         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
 997
 998         let events = nodes[0].node.get_and_clear_pending_events();
 999         assert_eq!(events.len(), 1);
1000         match events[0] {
1001                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1002                 _ => panic!("Unexpected event"),
1003         }
1004 }
1005
1006 #[test]
1007 fn raa_no_response_awaiting_raa_state() {
1008         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1009         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1010         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1011         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1012         let node_cfgs = create_node_cfgs(2);
1013         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1014         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1015         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1016
1017         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1018         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1019         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1020         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1021
1022         // Queue up two payments - one will be delivered right away, one immediately goes into the
1023         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1024         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1025         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1026         // generation during RAA while in monitor-update-failed state.
1027         nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1028         check_added_monitors!(nodes[0], 1);
1029         nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1030         check_added_monitors!(nodes[0], 0);
1031
1032         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1033         assert_eq!(events.len(), 1);
1034         let payment_event = SendEvent::from_event(events.pop().unwrap());
1035         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1036         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1037         check_added_monitors!(nodes[1], 1);
1038
1039         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1040         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1041         check_added_monitors!(nodes[0], 1);
1042         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1043         assert_eq!(events.len(), 1);
1044         let payment_event = SendEvent::from_event(events.pop().unwrap());
1045
1046         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1047         check_added_monitors!(nodes[0], 1);
1048         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1049
1050         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1051         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1052         // then restore channel monitor updates.
1053         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1054         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1055         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1056         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1057         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1058         check_added_monitors!(nodes[1], 1);
1059
1060         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1061         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1062         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1063         check_added_monitors!(nodes[1], 1);
1064
1065         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1066         nodes[1].node.test_restore_channel_monitor();
1067         // nodes[1] should be AwaitingRAA here!
1068         check_added_monitors!(nodes[1], 1);
1069         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1070         expect_pending_htlcs_forwardable!(nodes[1]);
1071         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1072
1073         // We send a third payment here, which is somewhat of a redundant test, but the
1074         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1075         // commitment transaction states) whereas here we can explicitly check for it.
1076         nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1077         check_added_monitors!(nodes[0], 0);
1078         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1079
1080         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1081         check_added_monitors!(nodes[0], 1);
1082         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1083         assert_eq!(events.len(), 1);
1084         let payment_event = SendEvent::from_event(events.pop().unwrap());
1085
1086         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1087         check_added_monitors!(nodes[0], 1);
1088         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1089
1090         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1091         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1092         check_added_monitors!(nodes[1], 1);
1093         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1094
1095         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1096         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1097         check_added_monitors!(nodes[1], 1);
1098         expect_pending_htlcs_forwardable!(nodes[1]);
1099         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1100         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1101
1102         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1103         check_added_monitors!(nodes[0], 1);
1104
1105         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1106         check_added_monitors!(nodes[0], 1);
1107         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1108
1109         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1110         check_added_monitors!(nodes[1], 1);
1111         expect_pending_htlcs_forwardable!(nodes[1]);
1112         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1113
1114         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1115         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1116         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1117 }
1118
1119 #[test]
1120 fn claim_while_disconnected_monitor_update_fail() {
1121         // Test for claiming a payment while disconnected and then having the resulting
1122         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1123         // contrived case for nodes with network instability.
1124         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1125         // code introduced a regression in this test (specifically, this caught a removal of the
1126         // channel_reestablish handling ensuring the order was sensical given the messages used).
1127         let node_cfgs = create_node_cfgs(2);
1128         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1129         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1130         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1131
1132         // Forward a payment for B to claim
1133         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1134
1135         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1136         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1137
1138         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1139         check_added_monitors!(nodes[1], 1);
1140
1141         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1142         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1143
1144         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1145         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1146
1147         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1148         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1149
1150         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1151         // update.
1152         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1153
1154         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1155         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1156         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1157         check_added_monitors!(nodes[1], 1);
1158         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1159
1160         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1161         // the monitor still failed
1162         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1163         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1164         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1165         check_added_monitors!(nodes[0], 1);
1166
1167         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1168         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1169         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1170         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1171         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1172         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1173         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1174
1175         // Now un-fail the monitor, which will result in B sending its original commitment update,
1176         // receiving the commitment update from A, and the resulting commitment dances.
1177         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1178         nodes[1].node.test_restore_channel_monitor();
1179         check_added_monitors!(nodes[1], 1);
1180
1181         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1182         assert_eq!(bs_msgs.len(), 2);
1183
1184         match bs_msgs[0] {
1185                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1186                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1187                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1188                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1189                         check_added_monitors!(nodes[0], 1);
1190
1191                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1192                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1193                         check_added_monitors!(nodes[1], 1);
1194                 },
1195                 _ => panic!("Unexpected event"),
1196         }
1197
1198         match bs_msgs[1] {
1199                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1200                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1201                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1202                         check_added_monitors!(nodes[0], 1);
1203                 },
1204                 _ => panic!("Unexpected event"),
1205         }
1206
1207         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1208
1209         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1210         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1211         check_added_monitors!(nodes[0], 1);
1212         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1213
1214         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1215         check_added_monitors!(nodes[1], 1);
1216         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1217         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1218         check_added_monitors!(nodes[1], 1);
1219
1220         expect_pending_htlcs_forwardable!(nodes[1]);
1221         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1222
1223         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1224         check_added_monitors!(nodes[0], 1);
1225
1226         let events = nodes[0].node.get_and_clear_pending_events();
1227         assert_eq!(events.len(), 1);
1228         match events[0] {
1229                 Event::PaymentSent { ref payment_preimage } => {
1230                         assert_eq!(*payment_preimage, payment_preimage_1);
1231                 },
1232                 _ => panic!("Unexpected event"),
1233         }
1234
1235         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1236 }
1237
1238 #[test]
1239 fn monitor_failed_no_reestablish_response() {
1240         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1241         // response to a commitment_signed.
1242         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1243         // debug_assert!() failure in channel_reestablish handling.
1244         let node_cfgs = create_node_cfgs(2);
1245         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1246         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1247         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1248
1249         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1250         // on receipt).
1251         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1252         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1253         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1254         check_added_monitors!(nodes[0], 1);
1255
1256         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1257         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1258         assert_eq!(events.len(), 1);
1259         let payment_event = SendEvent::from_event(events.pop().unwrap());
1260         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1261         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1262         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1263         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1264         check_added_monitors!(nodes[1], 1);
1265
1266         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1267         // is still failing to update monitors.
1268         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1269         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1270
1271         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1272         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1273
1274         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1275         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1276
1277         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1278         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1279
1280         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1281         nodes[1].node.test_restore_channel_monitor();
1282         check_added_monitors!(nodes[1], 1);
1283         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1284
1285         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1286         check_added_monitors!(nodes[0], 1);
1287         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1288         check_added_monitors!(nodes[0], 1);
1289
1290         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1291         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1292         check_added_monitors!(nodes[1], 1);
1293
1294         expect_pending_htlcs_forwardable!(nodes[1]);
1295         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1296
1297         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1298 }
1299
1300 #[test]
1301 fn first_message_on_recv_ordering() {
1302         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1303         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1304         // a commitment_signed which needs to send an RAA first.
1305         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1306         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1307         // response. To do this, we start routing two payments, with the final RAA for the first being
1308         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1309         // have no pending response but will want to send a RAA/CS (with the updates for the second
1310         // payment applied).
1311         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1312         let node_cfgs = create_node_cfgs(2);
1313         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1314         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1315         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1316
1317         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1318         // can deliver it and fail the monitor update.
1319         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1320         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1321         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1322         check_added_monitors!(nodes[0], 1);
1323
1324         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1325         assert_eq!(events.len(), 1);
1326         let payment_event = SendEvent::from_event(events.pop().unwrap());
1327         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1328         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1329         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1330         check_added_monitors!(nodes[1], 1);
1331         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1332
1333         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1334         check_added_monitors!(nodes[0], 1);
1335         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1336         check_added_monitors!(nodes[0], 1);
1337
1338         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1339
1340         // Route the second payment, generating an update_add_htlc/commitment_signed
1341         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1342         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1343         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1344         check_added_monitors!(nodes[0], 1);
1345         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1346         assert_eq!(events.len(), 1);
1347         let payment_event = SendEvent::from_event(events.pop().unwrap());
1348         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1349
1350         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1351
1352         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1353         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1354         // to the next message also tests resetting the delivery order.
1355         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1356         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1357         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1358         check_added_monitors!(nodes[1], 1);
1359
1360         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1361         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1362         // the appropriate HTLC acceptance).
1363         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1364         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1365         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1366         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1367
1368         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1369         nodes[1].node.test_restore_channel_monitor();
1370         check_added_monitors!(nodes[1], 1);
1371
1372         expect_pending_htlcs_forwardable!(nodes[1]);
1373         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1374
1375         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1376         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1377         check_added_monitors!(nodes[0], 1);
1378         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1379         check_added_monitors!(nodes[0], 1);
1380
1381         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1382         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1383         check_added_monitors!(nodes[1], 1);
1384
1385         expect_pending_htlcs_forwardable!(nodes[1]);
1386         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1387
1388         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1389         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1390 }
1391
1392 #[test]
1393 fn test_monitor_update_fail_claim() {
1394         // Basic test for monitor update failures when processing claim_funds calls.
1395         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1396         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1397         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1398         // updating and claim the payment on B.
1399         let node_cfgs = create_node_cfgs(3);
1400         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1401         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1402         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1403         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1404
1405         // Rebalance a bit so that we can send backwards from 3 to 2.
1406         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1407
1408         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1409
1410         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1411         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1412         check_added_monitors!(nodes[1], 1);
1413
1414         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1415         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1416         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1417         check_added_monitors!(nodes[2], 1);
1418
1419         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1420         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1421         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1422
1423         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1424         assert_eq!(events.len(), 1);
1425         let payment_event = SendEvent::from_event(events.pop().unwrap());
1426         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1427         let events = nodes[1].node.get_and_clear_pending_msg_events();
1428         assert_eq!(events.len(), 0);
1429         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1430         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1431
1432         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1433         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1434         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1435
1436         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1437         assert_eq!(msg_events.len(), 1);
1438         match msg_events[0] {
1439                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1440                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1441                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1442                 },
1443                 _ => panic!("Unexpected event"),
1444         }
1445
1446         let events = nodes[2].node.get_and_clear_pending_events();
1447         assert_eq!(events.len(), 1);
1448         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1449                 assert_eq!(payment_hash, payment_hash_2);
1450                 assert!(!rejected_by_dest);
1451         } else { panic!("Unexpected event!"); }
1452
1453         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1454         nodes[1].node.test_restore_channel_monitor();
1455         check_added_monitors!(nodes[1], 1);
1456
1457         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1458         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1459         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1460
1461         let events = nodes[0].node.get_and_clear_pending_events();
1462         assert_eq!(events.len(), 1);
1463         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1464                 assert_eq!(payment_preimage, payment_preimage_1);
1465         } else { panic!("Unexpected event!"); }
1466 }
1467
1468 #[test]
1469 fn test_monitor_update_on_pending_forwards() {
1470         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1471         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1472         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1473         // from C to A will be pending a forward to A.
1474         let node_cfgs = create_node_cfgs(3);
1475         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1476         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1477         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1478         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1479
1480         // Rebalance a bit so that we can send backwards from 3 to 1.
1481         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1482
1483         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1484         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1485         expect_pending_htlcs_forwardable!(nodes[2]);
1486         check_added_monitors!(nodes[2], 1);
1487
1488         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1489         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1490         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1491         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1492
1493         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1494         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1495         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1496         check_added_monitors!(nodes[2], 1);
1497
1498         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1499         assert_eq!(events.len(), 1);
1500         let payment_event = SendEvent::from_event(events.pop().unwrap());
1501         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1502         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1503
1504         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1505         expect_pending_htlcs_forwardable!(nodes[1]);
1506         check_added_monitors!(nodes[1], 1);
1507         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1508         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1509
1510         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1511         nodes[1].node.test_restore_channel_monitor();
1512         check_added_monitors!(nodes[1], 1);
1513
1514         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1515         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1516         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1517         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1518
1519         let events = nodes[0].node.get_and_clear_pending_events();
1520         assert_eq!(events.len(), 2);
1521         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1522                 assert_eq!(payment_hash, payment_hash_1);
1523                 assert!(rejected_by_dest);
1524         } else { panic!("Unexpected event!"); }
1525         match events[1] {
1526                 Event::PendingHTLCsForwardable { .. } => { },
1527                 _ => panic!("Unexpected event"),
1528         };
1529         nodes[0].node.process_pending_htlc_forwards();
1530         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1531
1532         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1533 }
1534
1535 #[test]
1536 fn monitor_update_claim_fail_no_response() {
1537         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1538         // to channel being AwaitingRAA).
1539         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1540         // code was broken.
1541         let node_cfgs = create_node_cfgs(2);
1542         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1543         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1544         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1545
1546         // Forward a payment for B to claim
1547         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1548
1549         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1550         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1551         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1552         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1553         check_added_monitors!(nodes[0], 1);
1554
1555         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1556         assert_eq!(events.len(), 1);
1557         let payment_event = SendEvent::from_event(events.pop().unwrap());
1558         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1559         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1560
1561         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1562         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1563         check_added_monitors!(nodes[1], 1);
1564         let events = nodes[1].node.get_and_clear_pending_msg_events();
1565         assert_eq!(events.len(), 0);
1566         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1567
1568         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1569         nodes[1].node.test_restore_channel_monitor();
1570         check_added_monitors!(nodes[1], 1);
1571         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1572
1573         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1574         check_added_monitors!(nodes[1], 1);
1575         expect_pending_htlcs_forwardable!(nodes[1]);
1576         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1577
1578         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1579         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1580         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1581
1582         let events = nodes[0].node.get_and_clear_pending_events();
1583         assert_eq!(events.len(), 1);
1584         match events[0] {
1585                 Event::PaymentSent { ref payment_preimage } => {
1586                         assert_eq!(*payment_preimage, payment_preimage_1);
1587                 },
1588                 _ => panic!("Unexpected event"),
1589         }
1590
1591         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1592 }
1593
1594 // Note that restore_between_fails with !fail_on_generate is useless
1595 // Also note that !fail_on_generate && !fail_on_signed is useless
1596 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1597 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1598 // restore_b_before_conf has no meaning if !confirm_a_first
1599 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1600         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1601         // the channel setup happily after the update is restored.
1602         let node_cfgs = create_node_cfgs(2);
1603         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1604         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1605
1606         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1607         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1608         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1609
1610         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1611
1612         if fail_on_generate {
1613                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1614         }
1615         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1616         check_added_monitors!(nodes[0], 1);
1617
1618         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1619         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1620         check_added_monitors!(nodes[1], 1);
1621
1622         if restore_between_fails {
1623                 assert!(fail_on_generate);
1624                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1625                 nodes[0].node.test_restore_channel_monitor();
1626                 check_added_monitors!(nodes[0], 1);
1627                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1628                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1629         }
1630
1631         if fail_on_signed {
1632                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1633         } else {
1634                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1635                 assert!(fail_on_generate); // Somebody has to fail
1636         }
1637         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1638         if fail_on_signed || !restore_between_fails {
1639                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1640                 if fail_on_generate && !restore_between_fails {
1641                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1642                         check_added_monitors!(nodes[0], 0);
1643                 } else {
1644                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1645                         check_added_monitors!(nodes[0], 1);
1646                 }
1647                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1648                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1649                 nodes[0].node.test_restore_channel_monitor();
1650         }
1651
1652         check_added_monitors!(nodes[0], 1);
1653
1654         let events = nodes[0].node.get_and_clear_pending_events();
1655         assert_eq!(events.len(), 1);
1656         match events[0] {
1657                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1658                         assert_eq!(user_channel_id, 43);
1659                         assert_eq!(*funding_txo, funding_output);
1660                 },
1661                 _ => panic!("Unexpected event"),
1662         };
1663
1664         if confirm_a_first {
1665                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1666                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1667         } else {
1668                 assert!(!restore_b_before_conf);
1669                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1670                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1671         }
1672
1673         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1674         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1675         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1676         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1677         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1678         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1679
1680         if !restore_b_before_conf {
1681                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1682                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1683                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1684         }
1685
1686         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1687         nodes[1].node.test_restore_channel_monitor();
1688         check_added_monitors!(nodes[1], 1);
1689
1690         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1691                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1692
1693                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1694                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1695                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1696         } else {
1697                 if restore_b_before_conf {
1698                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1699                 }
1700                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1701                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1702         };
1703         for node in nodes.iter() {
1704                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1705                 node.router.handle_channel_update(&as_update).unwrap();
1706                 node.router.handle_channel_update(&bs_update).unwrap();
1707         }
1708
1709         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1710         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1711 }
1712
1713 #[test]
1714 fn during_funding_monitor_fail() {
1715         do_during_funding_monitor_fail(false, false, true, true, true);
1716         do_during_funding_monitor_fail(true, false, true, false, false);
1717         do_during_funding_monitor_fail(true, true, true, true, false);
1718         do_during_funding_monitor_fail(true, true, false, false, false);
1719 }