_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let chanmon_cfgs = create_chanmon_cfgs(2);
  23         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  24         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  25         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  26         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  27
  28         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  29         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  30
  31         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  32         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  33         check_added_monitors!(nodes[0], 1);
  34
  35         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  36         assert_eq!(events_1.len(), 2);
  37         match events_1[0] {
  38                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  39                 _ => panic!("Unexpected event"),
  40         };
  41         match events_1[1] {
  42                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  43                 _ => panic!("Unexpected event"),
  44         };
  45
  46         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  47         // PaymentFailed event
  48
  49         assert_eq!(nodes[0].node.list_channels().len(), 0);
  50 }
  51
  52 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  53         // Test that we can recover from a simple temporary monitor update failure optionally with
  54         // a disconnect in between
  55         let chanmon_cfgs = create_chanmon_cfgs(2);
  56         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  57         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  58         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  59         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  60
  61         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  62         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  63
  64         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  65         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  66         check_added_monitors!(nodes[0], 1);
  67
  68         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  69         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  70         assert_eq!(nodes[0].node.list_channels().len(), 1);
  71
  72         if disconnect {
  73                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  74                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  75                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  76         }
  77
  78         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  79         nodes[0].node.test_restore_channel_monitor();
  80         check_added_monitors!(nodes[0], 1);
  81
  82         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  83         assert_eq!(events_2.len(), 1);
  84         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  85         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  86         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  87         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  88
  89         expect_pending_htlcs_forwardable!(nodes[1]);
  90
  91         let events_3 = nodes[1].node.get_and_clear_pending_events();
  92         assert_eq!(events_3.len(), 1);
  93         match events_3[0] {
  94                 Event::PaymentReceived { ref payment_hash, amt } => {
  95                         assert_eq!(payment_hash_1, *payment_hash);
  96                         assert_eq!(amt, 1000000);
  97                 },
  98                 _ => panic!("Unexpected event"),
  99         }
 100
 101         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 102
 103         // Now set it to failed again...
 104         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 105         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 106         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 107         check_added_monitors!(nodes[0], 1);
 108
 109         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 110         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 111         assert_eq!(nodes[0].node.list_channels().len(), 1);
 112
 113         if disconnect {
 114                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 115                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 116                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 117         }
 118
 119         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 120         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 121         nodes[0].node.test_restore_channel_monitor();
 122         check_added_monitors!(nodes[0], 1);
 123         check_closed_broadcast!(nodes[0], false);
 124
 125         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 126         // PaymentFailed event
 127
 128         assert_eq!(nodes[0].node.list_channels().len(), 0);
 129 }
 130
 131 #[test]
 132 fn test_simple_monitor_temporary_update_fail() {
 133         do_test_simple_monitor_temporary_update_fail(false);
 134         do_test_simple_monitor_temporary_update_fail(true);
 135 }
 136
 137 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 138         let disconnect_flags = 8 | 16;
 139
 140         // Test that we can recover from a temporary monitor update failure with some in-flight
 141         // HTLCs going on at the same time potentially with some disconnection thrown in.
 142         // * First we route a payment, then get a temporary monitor update failure when trying to
 143         //   route a second payment. We then claim the first payment.
 144         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 145         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 146         //   the ChannelMonitor on a watchtower).
 147         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 148         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 149         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 150         //   disconnect_count & !disconnect_flags is 0).
 151         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 152         //   through message sending, potentially disconnect/reconnecting multiple times based on
 153         //   disconnect_count, to get the update_fulfill_htlc through.
 154         // * We then walk through more message exchanges to get the original update_add_htlc
 155         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 156         //   disconnect/reconnecting based on disconnect_count.
 157         let chanmon_cfgs = create_chanmon_cfgs(2);
 158         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 159         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 160         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 161         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 162
 163         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 164
 165         // Now try to send a second payment which will fail to send
 166         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 167         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 168
 169         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 170         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 171         check_added_monitors!(nodes[0], 1);
 172
 173         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 174         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 175         assert_eq!(nodes[0].node.list_channels().len(), 1);
 176
 177         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 178         // but nodes[0] won't respond since it is frozen.
 179         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
 180         check_added_monitors!(nodes[1], 1);
 181         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 182         assert_eq!(events_2.len(), 1);
 183         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 184                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 185                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 186                         assert!(update_add_htlcs.is_empty());
 187                         assert_eq!(update_fulfill_htlcs.len(), 1);
 188                         assert!(update_fail_htlcs.is_empty());
 189                         assert!(update_fail_malformed_htlcs.is_empty());
 190                         assert!(update_fee.is_none());
 191
 192                         if (disconnect_count & 16) == 0 {
 193                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 194                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 195                                 assert_eq!(events_3.len(), 1);
 196                                 match events_3[0] {
 197                                         Event::PaymentSent { ref payment_preimage } => {
 198                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 199                                         },
 200                                         _ => panic!("Unexpected event"),
 201                                 }
 202
 203                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 204                                 check_added_monitors!(nodes[0], 1);
 205                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 206                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 207                         }
 208
 209                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 210                 },
 211                 _ => panic!("Unexpected event"),
 212         };
 213
 214         if disconnect_count & !disconnect_flags > 0 {
 215                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 216                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 217         }
 218
 219         // Now fix monitor updating...
 220         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 221         nodes[0].node.test_restore_channel_monitor();
 222         check_added_monitors!(nodes[0], 1);
 223
 224         macro_rules! disconnect_reconnect_peers { () => { {
 225                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 226                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 227
 228                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 229                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 230                 assert_eq!(reestablish_1.len(), 1);
 231                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 232                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 233                 assert_eq!(reestablish_2.len(), 1);
 234
 235                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 236                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 237                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 238                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 239
 240                 assert!(as_resp.0.is_none());
 241                 assert!(bs_resp.0.is_none());
 242
 243                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 244         } } }
 245
 246         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 247                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 248                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 249
 250                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 251                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 252                 assert_eq!(reestablish_1.len(), 1);
 253                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 254                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 255                 assert_eq!(reestablish_2.len(), 1);
 256
 257                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 258                 check_added_monitors!(nodes[0], 0);
 259                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 260                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 261                 check_added_monitors!(nodes[1], 0);
 262                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 263
 264                 assert!(as_resp.0.is_none());
 265                 assert!(bs_resp.0.is_none());
 266
 267                 assert!(bs_resp.1.is_none());
 268                 if (disconnect_count & 16) == 0 {
 269                         assert!(bs_resp.2.is_none());
 270
 271                         assert!(as_resp.1.is_some());
 272                         assert!(as_resp.2.is_some());
 273                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 274                 } else {
 275                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 276                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 277                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 278                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 279                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 280                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 281
 282                         assert!(as_resp.1.is_none());
 283
 284                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 285                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 286                         assert_eq!(events_3.len(), 1);
 287                         match events_3[0] {
 288                                 Event::PaymentSent { ref payment_preimage } => {
 289                                         assert_eq!(*payment_preimage, payment_preimage_1);
 290                                 },
 291                                 _ => panic!("Unexpected event"),
 292                         }
 293
 294                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 295                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 296                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 297                         check_added_monitors!(nodes[0], 1);
 298
 299                         as_resp.1 = Some(as_resp_raa);
 300                         bs_resp.2 = None;
 301                 }
 302
 303                 if disconnect_count & !disconnect_flags > 1 {
 304                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 305
 306                         if (disconnect_count & 16) == 0 {
 307                                 assert!(reestablish_1 == second_reestablish_1);
 308                                 assert!(reestablish_2 == second_reestablish_2);
 309                         }
 310                         assert!(as_resp == second_as_resp);
 311                         assert!(bs_resp == second_bs_resp);
 312                 }
 313
 314                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 315         } else {
 316                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 317                 assert_eq!(events_4.len(), 2);
 318                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 319                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 320                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 321                                 msg.clone()
 322                         },
 323                         _ => panic!("Unexpected event"),
 324                 })
 325         };
 326
 327         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 328
 329         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 330         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 331         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 332         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 333         check_added_monitors!(nodes[1], 1);
 334
 335         if disconnect_count & !disconnect_flags > 2 {
 336                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 337
 338                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 339                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 340
 341                 assert!(as_resp.2.is_none());
 342                 assert!(bs_resp.2.is_none());
 343         }
 344
 345         let as_commitment_update;
 346         let bs_second_commitment_update;
 347
 348         macro_rules! handle_bs_raa { () => {
 349                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 350                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 351                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 352                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 353                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 354                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 355                 assert!(as_commitment_update.update_fee.is_none());
 356                 check_added_monitors!(nodes[0], 1);
 357         } }
 358
 359         macro_rules! handle_initial_raa { () => {
 360                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 361                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 362                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 363                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 364                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 365                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 366                 assert!(bs_second_commitment_update.update_fee.is_none());
 367                 check_added_monitors!(nodes[1], 1);
 368         } }
 369
 370         if (disconnect_count & 8) == 0 {
 371                 handle_bs_raa!();
 372
 373                 if disconnect_count & !disconnect_flags > 3 {
 374                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 375
 376                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 377                         assert!(bs_resp.1.is_none());
 378
 379                         assert!(as_resp.2.unwrap() == as_commitment_update);
 380                         assert!(bs_resp.2.is_none());
 381
 382                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 383                 }
 384
 385                 handle_initial_raa!();
 386
 387                 if disconnect_count & !disconnect_flags > 4 {
 388                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 389
 390                         assert!(as_resp.1.is_none());
 391                         assert!(bs_resp.1.is_none());
 392
 393                         assert!(as_resp.2.unwrap() == as_commitment_update);
 394                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 395                 }
 396         } else {
 397                 handle_initial_raa!();
 398
 399                 if disconnect_count & !disconnect_flags > 3 {
 400                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 401
 402                         assert!(as_resp.1.is_none());
 403                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 404
 405                         assert!(as_resp.2.is_none());
 406                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 407
 408                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 409                 }
 410
 411                 handle_bs_raa!();
 412
 413                 if disconnect_count & !disconnect_flags > 4 {
 414                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 415
 416                         assert!(as_resp.1.is_none());
 417                         assert!(bs_resp.1.is_none());
 418
 419                         assert!(as_resp.2.unwrap() == as_commitment_update);
 420                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 421                 }
 422         }
 423
 424         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 425         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 426         // No commitment_signed so get_event_msg's assert(len == 1) passes
 427         check_added_monitors!(nodes[0], 1);
 428
 429         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 430         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 431         // No commitment_signed so get_event_msg's assert(len == 1) passes
 432         check_added_monitors!(nodes[1], 1);
 433
 434         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 435         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 436         check_added_monitors!(nodes[1], 1);
 437
 438         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 439         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 440         check_added_monitors!(nodes[0], 1);
 441
 442         expect_pending_htlcs_forwardable!(nodes[1]);
 443
 444         let events_5 = nodes[1].node.get_and_clear_pending_events();
 445         assert_eq!(events_5.len(), 1);
 446         match events_5[0] {
 447                 Event::PaymentReceived { ref payment_hash, amt } => {
 448                         assert_eq!(payment_hash_2, *payment_hash);
 449                         assert_eq!(amt, 1000000);
 450                 },
 451                 _ => panic!("Unexpected event"),
 452         }
 453
 454         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 455 }
 456
 457 #[test]
 458 fn test_monitor_temporary_update_fail_a() {
 459         do_test_monitor_temporary_update_fail(0);
 460         do_test_monitor_temporary_update_fail(1);
 461         do_test_monitor_temporary_update_fail(2);
 462         do_test_monitor_temporary_update_fail(3);
 463         do_test_monitor_temporary_update_fail(4);
 464         do_test_monitor_temporary_update_fail(5);
 465 }
 466
 467 #[test]
 468 fn test_monitor_temporary_update_fail_b() {
 469         do_test_monitor_temporary_update_fail(2 | 8);
 470         do_test_monitor_temporary_update_fail(3 | 8);
 471         do_test_monitor_temporary_update_fail(4 | 8);
 472         do_test_monitor_temporary_update_fail(5 | 8);
 473 }
 474
 475 #[test]
 476 fn test_monitor_temporary_update_fail_c() {
 477         do_test_monitor_temporary_update_fail(1 | 16);
 478         do_test_monitor_temporary_update_fail(2 | 16);
 479         do_test_monitor_temporary_update_fail(3 | 16);
 480         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 481         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 482 }
 483
 484 #[test]
 485 fn test_monitor_update_fail_cs() {
 486         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 487         let chanmon_cfgs = create_chanmon_cfgs(2);
 488         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 489         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 490         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 491         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 492
 493         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 494         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 495         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 496         check_added_monitors!(nodes[0], 1);
 497
 498         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 499         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 500
 501         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 502         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 503         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 504         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 505         check_added_monitors!(nodes[1], 1);
 506         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 507
 508         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 509         nodes[1].node.test_restore_channel_monitor();
 510         check_added_monitors!(nodes[1], 1);
 511         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 512         assert_eq!(responses.len(), 2);
 513
 514         match responses[0] {
 515                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 516                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 517                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 518                         check_added_monitors!(nodes[0], 1);
 519                 },
 520                 _ => panic!("Unexpected event"),
 521         }
 522         match responses[1] {
 523                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 524                         assert!(updates.update_add_htlcs.is_empty());
 525                         assert!(updates.update_fulfill_htlcs.is_empty());
 526                         assert!(updates.update_fail_htlcs.is_empty());
 527                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 528                         assert!(updates.update_fee.is_none());
 529                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 530
 531                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 532                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 533                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 534                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 535                         check_added_monitors!(nodes[0], 1);
 536                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 537                 },
 538                 _ => panic!("Unexpected event"),
 539         }
 540
 541         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 542         nodes[0].node.test_restore_channel_monitor();
 543         check_added_monitors!(nodes[0], 1);
 544
 545         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 546         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 547         check_added_monitors!(nodes[1], 1);
 548
 549         expect_pending_htlcs_forwardable!(nodes[1]);
 550
 551         let events = nodes[1].node.get_and_clear_pending_events();
 552         assert_eq!(events.len(), 1);
 553         match events[0] {
 554                 Event::PaymentReceived { payment_hash, amt } => {
 555                         assert_eq!(payment_hash, our_payment_hash);
 556                         assert_eq!(amt, 1000000);
 557                 },
 558                 _ => panic!("Unexpected event"),
 559         };
 560
 561         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 562 }
 563
 564 #[test]
 565 fn test_monitor_update_fail_no_rebroadcast() {
 566         // Tests handling of a monitor update failure when no message rebroadcasting on
 567         // test_restore_channel_monitor() is required. Backported from
 568         // chanmon_fail_consistency fuzz tests.
 569         let chanmon_cfgs = create_chanmon_cfgs(2);
 570         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 571         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 572         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 573         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 574
 575         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 576         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 577         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 578         check_added_monitors!(nodes[0], 1);
 579
 580         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 581         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 582         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 583
 584         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 585         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 586         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 587         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 588         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 589         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 590         check_added_monitors!(nodes[1], 1);
 591
 592         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 593         nodes[1].node.test_restore_channel_monitor();
 594         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 595         check_added_monitors!(nodes[1], 1);
 596         expect_pending_htlcs_forwardable!(nodes[1]);
 597
 598         let events = nodes[1].node.get_and_clear_pending_events();
 599         assert_eq!(events.len(), 1);
 600         match events[0] {
 601                 Event::PaymentReceived { payment_hash, .. } => {
 602                         assert_eq!(payment_hash, our_payment_hash);
 603                 },
 604                 _ => panic!("Unexpected event"),
 605         }
 606
 607         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 608 }
 609
 610 #[test]
 611 fn test_monitor_update_raa_while_paused() {
 612         // Tests handling of an RAA while monitor updating has already been marked failed.
 613         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 614         let chanmon_cfgs = create_chanmon_cfgs(2);
 615         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 616         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 617         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 618         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 619
 620         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 621
 622         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 623         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 624         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 625         check_added_monitors!(nodes[0], 1);
 626         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 627
 628         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 629         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 630         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 631         check_added_monitors!(nodes[1], 1);
 632         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 633
 634         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 635         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 636         check_added_monitors!(nodes[1], 1);
 637         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 638
 639         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 640         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 641         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 642         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 643         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 644         check_added_monitors!(nodes[0], 1);
 645
 646         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 647         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 648         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 649         check_added_monitors!(nodes[0], 1);
 650
 651         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 652         nodes[0].node.test_restore_channel_monitor();
 653         check_added_monitors!(nodes[0], 1);
 654
 655         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 656         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 657         check_added_monitors!(nodes[1], 1);
 658         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 659
 660         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 661         check_added_monitors!(nodes[1], 1);
 662         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 663
 664         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 665         check_added_monitors!(nodes[0], 1);
 666         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 667
 668         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 669         check_added_monitors!(nodes[0], 1);
 670         expect_pending_htlcs_forwardable!(nodes[0]);
 671         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 672
 673         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 674         check_added_monitors!(nodes[1], 1);
 675         expect_pending_htlcs_forwardable!(nodes[1]);
 676         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 677
 678         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 679         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 680 }
 681
 682 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 683         // Tests handling of a monitor update failure when processing an incoming RAA
 684         let chanmon_cfgs = create_chanmon_cfgs(3);
 685         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 686         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 687         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 688         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 689         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 690
 691         // Rebalance a bit so that we can send backwards from 2 to 1.
 692         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 693
 694         // Route a first payment that we'll fail backwards
 695         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 696
 697         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 698         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 699         expect_pending_htlcs_forwardable!(nodes[2]);
 700         check_added_monitors!(nodes[2], 1);
 701
 702         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 703         assert!(updates.update_add_htlcs.is_empty());
 704         assert!(updates.update_fulfill_htlcs.is_empty());
 705         assert_eq!(updates.update_fail_htlcs.len(), 1);
 706         assert!(updates.update_fail_malformed_htlcs.is_empty());
 707         assert!(updates.update_fee.is_none());
 708         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 709
 710         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 711         check_added_monitors!(nodes[0], 0);
 712
 713         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 714         // holding cell.
 715         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 716         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 717         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 718         check_added_monitors!(nodes[0], 1);
 719
 720         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 721         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 722         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 723
 724         expect_pending_htlcs_forwardable!(nodes[1]);
 725         check_added_monitors!(nodes[1], 0);
 726         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 727
 728         // Now fail monitor updating.
 729         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 730         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 731         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 732         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 733         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 734         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 735         check_added_monitors!(nodes[1], 1);
 736
 737         // Attempt to forward a third payment but fail due to the second channel being unavailable
 738         // for forwarding.
 739
 740         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 741         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 742         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 743         check_added_monitors!(nodes[0], 1);
 744
 745         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 746         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 747         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 748         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 749         check_added_monitors!(nodes[1], 0);
 750
 751         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 752         assert_eq!(events_2.len(), 1);
 753         match events_2.remove(0) {
 754                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 755                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 756                         assert!(updates.update_fulfill_htlcs.is_empty());
 757                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 758                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 759                         assert!(updates.update_add_htlcs.is_empty());
 760                         assert!(updates.update_fee.is_none());
 761
 762                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 763                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 764
 765                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 766                         assert_eq!(msg_events.len(), 1);
 767                         match msg_events[0] {
 768                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 769                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 770                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 771                                 },
 772                                 _ => panic!("Unexpected event"),
 773                         }
 774
 775                         let events = nodes[0].node.get_and_clear_pending_events();
 776                         assert_eq!(events.len(), 1);
 777                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 778                                 assert_eq!(payment_hash, payment_hash_3);
 779                                 assert!(!rejected_by_dest);
 780                         } else { panic!("Unexpected event!"); }
 781                 },
 782                 _ => panic!("Unexpected event type!"),
 783         };
 784
 785         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 786                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 787                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 788                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 789                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 790                 check_added_monitors!(nodes[2], 1);
 791
 792                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 793                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 794                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 795                 check_added_monitors!(nodes[1], 1);
 796                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 797                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 798                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 799                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 800                 (Some(payment_preimage_4), Some(payment_hash_4))
 801         } else { (None, None) };
 802
 803         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 804         // update_add update.
 805         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 806         nodes[1].node.test_restore_channel_monitor();
 807         check_added_monitors!(nodes[1], 1);
 808         expect_pending_htlcs_forwardable!(nodes[1]);
 809         check_added_monitors!(nodes[1], 1);
 810
 811         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 812         if test_ignore_second_cs {
 813                 assert_eq!(events_3.len(), 3);
 814         } else {
 815                 assert_eq!(events_3.len(), 2);
 816         }
 817
 818         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 819         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 820         let messages_a = match events_3.pop().unwrap() {
 821                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 822                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 823                         assert!(updates.update_fulfill_htlcs.is_empty());
 824                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 825                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 826                         assert!(updates.update_add_htlcs.is_empty());
 827                         assert!(updates.update_fee.is_none());
 828                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 829                 },
 830                 _ => panic!("Unexpected event type!"),
 831         };
 832         let raa = if test_ignore_second_cs {
 833                 match events_3.remove(1) {
 834                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 835                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 836                                 Some(msg.clone())
 837                         },
 838                         _ => panic!("Unexpected event"),
 839                 }
 840         } else { None };
 841         let send_event_b = SendEvent::from_event(events_3.remove(0));
 842         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 843
 844         // Now deliver the new messages...
 845
 846         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 847         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 848         let events_4 = nodes[0].node.get_and_clear_pending_events();
 849         assert_eq!(events_4.len(), 1);
 850         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 851                 assert_eq!(payment_hash, payment_hash_1);
 852                 assert!(rejected_by_dest);
 853         } else { panic!("Unexpected event!"); }
 854
 855         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 856         if test_ignore_second_cs {
 857                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 858                 check_added_monitors!(nodes[2], 1);
 859                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 860                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 861                 check_added_monitors!(nodes[2], 1);
 862                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 863                 assert!(bs_cs.update_add_htlcs.is_empty());
 864                 assert!(bs_cs.update_fail_htlcs.is_empty());
 865                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 866                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 867                 assert!(bs_cs.update_fee.is_none());
 868
 869                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 870                 check_added_monitors!(nodes[1], 1);
 871                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 872                 assert!(as_cs.update_add_htlcs.is_empty());
 873                 assert!(as_cs.update_fail_htlcs.is_empty());
 874                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 875                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 876                 assert!(as_cs.update_fee.is_none());
 877
 878                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 879                 check_added_monitors!(nodes[1], 1);
 880                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 881
 882                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 883                 check_added_monitors!(nodes[2], 1);
 884                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 885
 886                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 887                 check_added_monitors!(nodes[2], 1);
 888                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 889
 890                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 891                 check_added_monitors!(nodes[1], 1);
 892                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 893         } else {
 894                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 895         }
 896
 897         expect_pending_htlcs_forwardable!(nodes[2]);
 898
 899         let events_6 = nodes[2].node.get_and_clear_pending_events();
 900         assert_eq!(events_6.len(), 1);
 901         match events_6[0] {
 902                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 903                 _ => panic!("Unexpected event"),
 904         };
 905
 906         if test_ignore_second_cs {
 907                 expect_pending_htlcs_forwardable!(nodes[1]);
 908                 check_added_monitors!(nodes[1], 1);
 909
 910                 send_event = SendEvent::from_node(&nodes[1]);
 911                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 912                 assert_eq!(send_event.msgs.len(), 1);
 913                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 914                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 915
 916                 expect_pending_htlcs_forwardable!(nodes[0]);
 917
 918                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 919                 assert_eq!(events_9.len(), 1);
 920                 match events_9[0] {
 921                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 922                         _ => panic!("Unexpected event"),
 923                 };
 924                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 925         }
 926
 927         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 928 }
 929
 930 #[test]
 931 fn test_monitor_update_fail_raa() {
 932         do_test_monitor_update_fail_raa(false);
 933         do_test_monitor_update_fail_raa(true);
 934 }
 935
 936 #[test]
 937 fn test_monitor_update_fail_reestablish() {
 938         // Simple test for message retransmission after monitor update failure on
 939         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 940         // HTLCs).
 941         let chanmon_cfgs = create_chanmon_cfgs(3);
 942         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 943         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 944         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 945         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 946         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 947
 948         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 949
 950         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 951         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 952
 953         assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
 954         check_added_monitors!(nodes[2], 1);
 955         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 956         assert!(updates.update_add_htlcs.is_empty());
 957         assert!(updates.update_fail_htlcs.is_empty());
 958         assert!(updates.update_fail_malformed_htlcs.is_empty());
 959         assert!(updates.update_fee.is_none());
 960         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 961         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 962         check_added_monitors!(nodes[1], 1);
 963         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 964         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 965
 966         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 967         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 968         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 969
 970         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 971         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 972
 973         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 974
 975         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 976         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 977         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 978         check_added_monitors!(nodes[1], 1);
 979
 980         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 981         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 982
 983         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 984         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 985
 986         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 987         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 988
 989         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 990
 991         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 992         check_added_monitors!(nodes[1], 0);
 993         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 994
 995         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 996         nodes[1].node.test_restore_channel_monitor();
 997         check_added_monitors!(nodes[1], 1);
 998
 999         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1000         assert!(updates.update_add_htlcs.is_empty());
1001         assert!(updates.update_fail_htlcs.is_empty());
1002         assert!(updates.update_fail_malformed_htlcs.is_empty());
1003         assert!(updates.update_fee.is_none());
1004         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1005         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1006         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1007
1008         let events = nodes[0].node.get_and_clear_pending_events();
1009         assert_eq!(events.len(), 1);
1010         match events[0] {
1011                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1012                 _ => panic!("Unexpected event"),
1013         }
1014 }
1015
1016 #[test]
1017 fn raa_no_response_awaiting_raa_state() {
1018         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1019         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1020         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1021         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1022         let chanmon_cfgs = create_chanmon_cfgs(2);
1023         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1024         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1025         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1026         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1027
1028         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1029         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1030         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1031         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1032
1033         // Queue up two payments - one will be delivered right away, one immediately goes into the
1034         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1035         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1036         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1037         // generation during RAA while in monitor-update-failed state.
1038         nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1039         check_added_monitors!(nodes[0], 1);
1040         nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1041         check_added_monitors!(nodes[0], 0);
1042
1043         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1044         assert_eq!(events.len(), 1);
1045         let payment_event = SendEvent::from_event(events.pop().unwrap());
1046         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1047         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1048         check_added_monitors!(nodes[1], 1);
1049
1050         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1051         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1052         check_added_monitors!(nodes[0], 1);
1053         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1054         assert_eq!(events.len(), 1);
1055         let payment_event = SendEvent::from_event(events.pop().unwrap());
1056
1057         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1058         check_added_monitors!(nodes[0], 1);
1059         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1060
1061         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1062         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1063         // then restore channel monitor updates.
1064         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1065         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1066         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1067         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1068         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1069         check_added_monitors!(nodes[1], 1);
1070
1071         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1072         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1073         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1074         check_added_monitors!(nodes[1], 1);
1075
1076         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1077         nodes[1].node.test_restore_channel_monitor();
1078         // nodes[1] should be AwaitingRAA here!
1079         check_added_monitors!(nodes[1], 1);
1080         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1081         expect_pending_htlcs_forwardable!(nodes[1]);
1082         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1083
1084         // We send a third payment here, which is somewhat of a redundant test, but the
1085         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1086         // commitment transaction states) whereas here we can explicitly check for it.
1087         nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1088         check_added_monitors!(nodes[0], 0);
1089         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1090
1091         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1092         check_added_monitors!(nodes[0], 1);
1093         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1094         assert_eq!(events.len(), 1);
1095         let payment_event = SendEvent::from_event(events.pop().unwrap());
1096
1097         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1098         check_added_monitors!(nodes[0], 1);
1099         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1100
1101         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1102         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1103         check_added_monitors!(nodes[1], 1);
1104         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1105
1106         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1107         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1108         check_added_monitors!(nodes[1], 1);
1109         expect_pending_htlcs_forwardable!(nodes[1]);
1110         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1111         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1112
1113         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1114         check_added_monitors!(nodes[0], 1);
1115
1116         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1117         check_added_monitors!(nodes[0], 1);
1118         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1119
1120         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1121         check_added_monitors!(nodes[1], 1);
1122         expect_pending_htlcs_forwardable!(nodes[1]);
1123         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1124
1125         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1126         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1127         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1128 }
1129
1130 #[test]
1131 fn claim_while_disconnected_monitor_update_fail() {
1132         // Test for claiming a payment while disconnected and then having the resulting
1133         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1134         // contrived case for nodes with network instability.
1135         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1136         // code introduced a regression in this test (specifically, this caught a removal of the
1137         // channel_reestablish handling ensuring the order was sensical given the messages used).
1138         let chanmon_cfgs = create_chanmon_cfgs(2);
1139         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1140         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1141         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1142         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1143
1144         // Forward a payment for B to claim
1145         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1146
1147         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1148         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1149
1150         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1151         check_added_monitors!(nodes[1], 1);
1152
1153         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1154         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1155
1156         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1157         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1158
1159         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1160         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1161
1162         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1163         // update.
1164         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1165
1166         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1167         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1168         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1169         check_added_monitors!(nodes[1], 1);
1170         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1171
1172         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1173         // the monitor still failed
1174         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1175         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1176         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1177         check_added_monitors!(nodes[0], 1);
1178
1179         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1180         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1181         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1182         check_added_monitors!(nodes[1], 1);
1183         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1184         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1185         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1186         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1187
1188         // Now un-fail the monitor, which will result in B sending its original commitment update,
1189         // receiving the commitment update from A, and the resulting commitment dances.
1190         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1191         nodes[1].node.test_restore_channel_monitor();
1192         check_added_monitors!(nodes[1], 1);
1193
1194         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1195         assert_eq!(bs_msgs.len(), 2);
1196
1197         match bs_msgs[0] {
1198                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1199                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1200                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1201                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1202                         check_added_monitors!(nodes[0], 1);
1203
1204                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1205                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1206                         check_added_monitors!(nodes[1], 1);
1207                 },
1208                 _ => panic!("Unexpected event"),
1209         }
1210
1211         match bs_msgs[1] {
1212                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1213                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1214                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1215                         check_added_monitors!(nodes[0], 1);
1216                 },
1217                 _ => panic!("Unexpected event"),
1218         }
1219
1220         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1221
1222         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1223         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1224         check_added_monitors!(nodes[0], 1);
1225         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1226
1227         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1228         check_added_monitors!(nodes[1], 1);
1229         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1230         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1231         check_added_monitors!(nodes[1], 1);
1232
1233         expect_pending_htlcs_forwardable!(nodes[1]);
1234         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1235
1236         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1237         check_added_monitors!(nodes[0], 1);
1238
1239         let events = nodes[0].node.get_and_clear_pending_events();
1240         assert_eq!(events.len(), 1);
1241         match events[0] {
1242                 Event::PaymentSent { ref payment_preimage } => {
1243                         assert_eq!(*payment_preimage, payment_preimage_1);
1244                 },
1245                 _ => panic!("Unexpected event"),
1246         }
1247
1248         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1249 }
1250
1251 #[test]
1252 fn monitor_failed_no_reestablish_response() {
1253         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1254         // response to a commitment_signed.
1255         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1256         // debug_assert!() failure in channel_reestablish handling.
1257         let chanmon_cfgs = create_chanmon_cfgs(2);
1258         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1259         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1260         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1261         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1262
1263         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1264         // on receipt).
1265         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1266         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1267         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1268         check_added_monitors!(nodes[0], 1);
1269
1270         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1271         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1272         assert_eq!(events.len(), 1);
1273         let payment_event = SendEvent::from_event(events.pop().unwrap());
1274         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1275         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1276         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1277         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1278         check_added_monitors!(nodes[1], 1);
1279
1280         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1281         // is still failing to update monitors.
1282         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1283         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1284
1285         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1286         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1287
1288         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1289         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1290
1291         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1292         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1293
1294         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1295         nodes[1].node.test_restore_channel_monitor();
1296         check_added_monitors!(nodes[1], 1);
1297         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1298
1299         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1300         check_added_monitors!(nodes[0], 1);
1301         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1302         check_added_monitors!(nodes[0], 1);
1303
1304         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1305         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1306         check_added_monitors!(nodes[1], 1);
1307
1308         expect_pending_htlcs_forwardable!(nodes[1]);
1309         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1310
1311         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1312 }
1313
1314 #[test]
1315 fn first_message_on_recv_ordering() {
1316         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1317         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1318         // a commitment_signed which needs to send an RAA first.
1319         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1320         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1321         // response. To do this, we start routing two payments, with the final RAA for the first being
1322         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1323         // have no pending response but will want to send a RAA/CS (with the updates for the second
1324         // payment applied).
1325         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1326         let chanmon_cfgs = create_chanmon_cfgs(2);
1327         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1328         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1329         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1330         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1331
1332         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1333         // can deliver it and fail the monitor update.
1334         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1335         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1336         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1337         check_added_monitors!(nodes[0], 1);
1338
1339         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1340         assert_eq!(events.len(), 1);
1341         let payment_event = SendEvent::from_event(events.pop().unwrap());
1342         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1343         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1344         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1345         check_added_monitors!(nodes[1], 1);
1346         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1347
1348         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1349         check_added_monitors!(nodes[0], 1);
1350         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1351         check_added_monitors!(nodes[0], 1);
1352
1353         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1354
1355         // Route the second payment, generating an update_add_htlc/commitment_signed
1356         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1357         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1358         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1359         check_added_monitors!(nodes[0], 1);
1360         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1361         assert_eq!(events.len(), 1);
1362         let payment_event = SendEvent::from_event(events.pop().unwrap());
1363         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1364
1365         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1366
1367         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1368         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1369         // to the next message also tests resetting the delivery order.
1370         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1371         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1372         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1373         check_added_monitors!(nodes[1], 1);
1374
1375         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1376         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1377         // the appropriate HTLC acceptance).
1378         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1379         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1380         check_added_monitors!(nodes[1], 1);
1381         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1382         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1383
1384         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1385         nodes[1].node.test_restore_channel_monitor();
1386         check_added_monitors!(nodes[1], 1);
1387
1388         expect_pending_htlcs_forwardable!(nodes[1]);
1389         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1390
1391         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1392         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1393         check_added_monitors!(nodes[0], 1);
1394         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1395         check_added_monitors!(nodes[0], 1);
1396
1397         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1398         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1399         check_added_monitors!(nodes[1], 1);
1400
1401         expect_pending_htlcs_forwardable!(nodes[1]);
1402         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1403
1404         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1405         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1406 }
1407
1408 #[test]
1409 fn test_monitor_update_fail_claim() {
1410         // Basic test for monitor update failures when processing claim_funds calls.
1411         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1412         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1413         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1414         // updating and claim the payment on B.
1415         let chanmon_cfgs = create_chanmon_cfgs(3);
1416         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1417         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1418         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1419         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1420         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1421
1422         // Rebalance a bit so that we can send backwards from 3 to 2.
1423         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1424
1425         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1426
1427         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1428         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1429         check_added_monitors!(nodes[1], 1);
1430
1431         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1432         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1433         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1434         check_added_monitors!(nodes[2], 1);
1435
1436         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1437         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1438         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1439
1440         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1441         assert_eq!(events.len(), 1);
1442         let payment_event = SendEvent::from_event(events.pop().unwrap());
1443         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1444         let events = nodes[1].node.get_and_clear_pending_msg_events();
1445         assert_eq!(events.len(), 0);
1446         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1447         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1448
1449         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1450         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1451         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1452
1453         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1454         assert_eq!(msg_events.len(), 1);
1455         match msg_events[0] {
1456                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1457                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1458                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1459                 },
1460                 _ => panic!("Unexpected event"),
1461         }
1462
1463         let events = nodes[2].node.get_and_clear_pending_events();
1464         assert_eq!(events.len(), 1);
1465         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1466                 assert_eq!(payment_hash, payment_hash_2);
1467                 assert!(!rejected_by_dest);
1468         } else { panic!("Unexpected event!"); }
1469
1470         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1471         nodes[1].node.test_restore_channel_monitor();
1472         check_added_monitors!(nodes[1], 1);
1473
1474         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1475         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1476         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1477
1478         let events = nodes[0].node.get_and_clear_pending_events();
1479         assert_eq!(events.len(), 1);
1480         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1481                 assert_eq!(payment_preimage, payment_preimage_1);
1482         } else { panic!("Unexpected event!"); }
1483 }
1484
1485 #[test]
1486 fn test_monitor_update_on_pending_forwards() {
1487         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1488         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1489         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1490         // from C to A will be pending a forward to A.
1491         let chanmon_cfgs = create_chanmon_cfgs(3);
1492         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1493         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1494         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1495         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1496         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1497
1498         // Rebalance a bit so that we can send backwards from 3 to 1.
1499         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1500
1501         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1502         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1503         expect_pending_htlcs_forwardable!(nodes[2]);
1504         check_added_monitors!(nodes[2], 1);
1505
1506         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1507         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1508         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1509         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1510
1511         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1512         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1513         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1514         check_added_monitors!(nodes[2], 1);
1515
1516         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1517         assert_eq!(events.len(), 1);
1518         let payment_event = SendEvent::from_event(events.pop().unwrap());
1519         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1520         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1521
1522         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1523         expect_pending_htlcs_forwardable!(nodes[1]);
1524         check_added_monitors!(nodes[1], 1);
1525         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1526         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1527
1528         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1529         nodes[1].node.test_restore_channel_monitor();
1530         check_added_monitors!(nodes[1], 1);
1531
1532         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1533         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1534         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1535         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1536
1537         let events = nodes[0].node.get_and_clear_pending_events();
1538         assert_eq!(events.len(), 2);
1539         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1540                 assert_eq!(payment_hash, payment_hash_1);
1541                 assert!(rejected_by_dest);
1542         } else { panic!("Unexpected event!"); }
1543         match events[1] {
1544                 Event::PendingHTLCsForwardable { .. } => { },
1545                 _ => panic!("Unexpected event"),
1546         };
1547         nodes[0].node.process_pending_htlc_forwards();
1548         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1549
1550         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1551 }
1552
1553 #[test]
1554 fn monitor_update_claim_fail_no_response() {
1555         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1556         // to channel being AwaitingRAA).
1557         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1558         // code was broken.
1559         let chanmon_cfgs = create_chanmon_cfgs(2);
1560         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1561         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1562         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1563         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1564
1565         // Forward a payment for B to claim
1566         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1567
1568         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1569         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1570         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1571         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1572         check_added_monitors!(nodes[0], 1);
1573
1574         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1575         assert_eq!(events.len(), 1);
1576         let payment_event = SendEvent::from_event(events.pop().unwrap());
1577         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1578         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1579
1580         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1581         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1582         check_added_monitors!(nodes[1], 1);
1583         let events = nodes[1].node.get_and_clear_pending_msg_events();
1584         assert_eq!(events.len(), 0);
1585         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1586
1587         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1588         nodes[1].node.test_restore_channel_monitor();
1589         check_added_monitors!(nodes[1], 1);
1590         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1591
1592         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1593         check_added_monitors!(nodes[1], 1);
1594         expect_pending_htlcs_forwardable!(nodes[1]);
1595         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1596
1597         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1598         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1599         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1600
1601         let events = nodes[0].node.get_and_clear_pending_events();
1602         assert_eq!(events.len(), 1);
1603         match events[0] {
1604                 Event::PaymentSent { ref payment_preimage } => {
1605                         assert_eq!(*payment_preimage, payment_preimage_1);
1606                 },
1607                 _ => panic!("Unexpected event"),
1608         }
1609
1610         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1611 }
1612
1613 // Note that restore_between_fails with !fail_on_generate is useless
1614 // Also note that !fail_on_generate && !fail_on_signed is useless
1615 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1616 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1617 // restore_b_before_conf has no meaning if !confirm_a_first
1618 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1619         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1620         // the channel setup happily after the update is restored.
1621         let chanmon_cfgs = create_chanmon_cfgs(2);
1622         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1623         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1624         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1625
1626         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1627         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1628         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1629
1630         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1631
1632         if fail_on_generate {
1633                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1634         }
1635         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1636         check_added_monitors!(nodes[0], 1);
1637
1638         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1639         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1640         check_added_monitors!(nodes[1], 1);
1641
1642         if restore_between_fails {
1643                 assert!(fail_on_generate);
1644                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1645                 nodes[0].node.test_restore_channel_monitor();
1646                 check_added_monitors!(nodes[0], 1);
1647                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1648                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1649         }
1650
1651         if fail_on_signed {
1652                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1653         } else {
1654                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1655                 assert!(fail_on_generate); // Somebody has to fail
1656         }
1657         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1658         if fail_on_signed || !restore_between_fails {
1659                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1660                 if fail_on_generate && !restore_between_fails {
1661                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1662                         check_added_monitors!(nodes[0], 1);
1663                 } else {
1664                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1665                         check_added_monitors!(nodes[0], 1);
1666                 }
1667                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1668                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1669                 nodes[0].node.test_restore_channel_monitor();
1670         }
1671
1672         check_added_monitors!(nodes[0], 1);
1673
1674         let events = nodes[0].node.get_and_clear_pending_events();
1675         assert_eq!(events.len(), 1);
1676         match events[0] {
1677                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1678                         assert_eq!(user_channel_id, 43);
1679                         assert_eq!(*funding_txo, funding_output);
1680                 },
1681                 _ => panic!("Unexpected event"),
1682         };
1683
1684         if confirm_a_first {
1685                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1686                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1687         } else {
1688                 assert!(!restore_b_before_conf);
1689                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1690                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1691         }
1692
1693         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1694         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1695         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1696         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1697         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1698         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1699
1700         if !restore_b_before_conf {
1701                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1702                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1703                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1704         }
1705
1706         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1707         nodes[1].node.test_restore_channel_monitor();
1708         check_added_monitors!(nodes[1], 1);
1709
1710         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1711                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1712
1713                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1714                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1715                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1716         } else {
1717                 if restore_b_before_conf {
1718                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1719                 }
1720                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1721                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1722         };
1723         for node in nodes.iter() {
1724                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1725                 node.router.handle_channel_update(&as_update).unwrap();
1726                 node.router.handle_channel_update(&bs_update).unwrap();
1727         }
1728
1729         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1730         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1731 }
1732
1733 #[test]
1734 fn during_funding_monitor_fail() {
1735         do_during_funding_monitor_fail(false, false, true, true, true);
1736         do_during_funding_monitor_fail(true, false, true, false, false);
1737         do_during_funding_monitor_fail(true, true, true, true, false);
1738         do_during_funding_monitor_fail(true, true, false, false, false);
1739 }