_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   7 use ln::channelmonitor::ChannelMonitorUpdateErr;
   8 use ln::features::InitFeatures;
   9 use ln::msgs;
  10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  12 use util::errors::APIError;
  13
  14 use bitcoin_hashes::sha256::Hash as Sha256;
  15 use bitcoin_hashes::Hash;
  16
  17 use ln::functional_test_utils::*;
  18
  19 #[test]
  20 fn test_simple_monitor_permanent_update_fail() {
  21         // Test that we handle a simple permanent monitor update failure
  22         let chanmon_cfgs = create_chanmon_cfgs(2);
  23         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  24         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  25         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  26         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  27
  28         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  29         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  30
  31         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  32         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  33         check_added_monitors!(nodes[0], 1);
  34
  35         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  36         assert_eq!(events_1.len(), 2);
  37         match events_1[0] {
  38                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  39                 _ => panic!("Unexpected event"),
  40         };
  41         match events_1[1] {
  42                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  43                 _ => panic!("Unexpected event"),
  44         };
  45
  46         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  47         // PaymentFailed event
  48
  49         assert_eq!(nodes[0].node.list_channels().len(), 0);
  50 }
  51
  52 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  53         // Test that we can recover from a simple temporary monitor update failure optionally with
  54         // a disconnect in between
  55         let chanmon_cfgs = create_chanmon_cfgs(2);
  56         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  57         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  58         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  59         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  60
  61         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  62         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  63
  64         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  65         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  66         check_added_monitors!(nodes[0], 1);
  67
  68         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  69         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  70         assert_eq!(nodes[0].node.list_channels().len(), 1);
  71
  72         if disconnect {
  73                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  74                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  75                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  76         }
  77
  78         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  79         nodes[0].node.test_restore_channel_monitor();
  80         check_added_monitors!(nodes[0], 1);
  81
  82         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  83         assert_eq!(events_2.len(), 1);
  84         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  85         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  86         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  87         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  88
  89         expect_pending_htlcs_forwardable!(nodes[1]);
  90
  91         let events_3 = nodes[1].node.get_and_clear_pending_events();
  92         assert_eq!(events_3.len(), 1);
  93         match events_3[0] {
  94                 Event::PaymentReceived { ref payment_hash, amt } => {
  95                         assert_eq!(payment_hash_1, *payment_hash);
  96                         assert_eq!(amt, 1000000);
  97                 },
  98                 _ => panic!("Unexpected event"),
  99         }
 100
 101         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 102
 103         // Now set it to failed again...
 104         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 105         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 106         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 107         check_added_monitors!(nodes[0], 1);
 108
 109         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 110         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 111         assert_eq!(nodes[0].node.list_channels().len(), 1);
 112
 113         if disconnect {
 114                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 115                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 116                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 117         }
 118
 119         // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
 120         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
 121         nodes[0].node.test_restore_channel_monitor();
 122         check_added_monitors!(nodes[0], 1);
 123         check_closed_broadcast!(nodes[0], false);
 124
 125         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 126         // PaymentFailed event
 127
 128         assert_eq!(nodes[0].node.list_channels().len(), 0);
 129 }
 130
 131 #[test]
 132 fn test_simple_monitor_temporary_update_fail() {
 133         do_test_simple_monitor_temporary_update_fail(false);
 134         do_test_simple_monitor_temporary_update_fail(true);
 135 }
 136
 137 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 138         let disconnect_flags = 8 | 16;
 139
 140         // Test that we can recover from a temporary monitor update failure with some in-flight
 141         // HTLCs going on at the same time potentially with some disconnection thrown in.
 142         // * First we route a payment, then get a temporary monitor update failure when trying to
 143         //   route a second payment. We then claim the first payment.
 144         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 145         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 146         //   the ChannelMonitor on a watchtower).
 147         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 148         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 149         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 150         //   disconnect_count & !disconnect_flags is 0).
 151         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 152         //   through message sending, potentially disconnect/reconnecting multiple times based on
 153         //   disconnect_count, to get the update_fulfill_htlc through.
 154         // * We then walk through more message exchanges to get the original update_add_htlc
 155         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 156         //   disconnect/reconnecting based on disconnect_count.
 157         let chanmon_cfgs = create_chanmon_cfgs(2);
 158         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 159         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 160         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 161         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 162
 163         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 164
 165         // Now try to send a second payment which will fail to send
 166         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 167         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 168
 169         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 170         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 171         check_added_monitors!(nodes[0], 1);
 172
 173         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 174         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 175         assert_eq!(nodes[0].node.list_channels().len(), 1);
 176
 177         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 178         // but nodes[0] won't respond since it is frozen.
 179         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
 180         check_added_monitors!(nodes[1], 1);
 181         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 182         assert_eq!(events_2.len(), 1);
 183         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 184                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 185                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 186                         assert!(update_add_htlcs.is_empty());
 187                         assert_eq!(update_fulfill_htlcs.len(), 1);
 188                         assert!(update_fail_htlcs.is_empty());
 189                         assert!(update_fail_malformed_htlcs.is_empty());
 190                         assert!(update_fee.is_none());
 191
 192                         if (disconnect_count & 16) == 0 {
 193                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 194                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 195                                 assert_eq!(events_3.len(), 1);
 196                                 match events_3[0] {
 197                                         Event::PaymentSent { ref payment_preimage } => {
 198                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 199                                         },
 200                                         _ => panic!("Unexpected event"),
 201                                 }
 202
 203                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 204                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 205                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 206                         }
 207
 208                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 209                 },
 210                 _ => panic!("Unexpected event"),
 211         };
 212
 213         if disconnect_count & !disconnect_flags > 0 {
 214                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 215                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 216         }
 217
 218         // Now fix monitor updating...
 219         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 220         nodes[0].node.test_restore_channel_monitor();
 221         check_added_monitors!(nodes[0], 1);
 222
 223         macro_rules! disconnect_reconnect_peers { () => { {
 224                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 225                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 226
 227                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 228                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 229                 assert_eq!(reestablish_1.len(), 1);
 230                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 231                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 232                 assert_eq!(reestablish_2.len(), 1);
 233
 234                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 235                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 236                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 237                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 238
 239                 assert!(as_resp.0.is_none());
 240                 assert!(bs_resp.0.is_none());
 241
 242                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 243         } } }
 244
 245         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 246                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 247                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 248
 249                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 250                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 251                 assert_eq!(reestablish_1.len(), 1);
 252                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 253                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 254                 assert_eq!(reestablish_2.len(), 1);
 255
 256                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 257                 check_added_monitors!(nodes[0], 0);
 258                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 259                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 260                 check_added_monitors!(nodes[1], 0);
 261                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 262
 263                 assert!(as_resp.0.is_none());
 264                 assert!(bs_resp.0.is_none());
 265
 266                 assert!(bs_resp.1.is_none());
 267                 if (disconnect_count & 16) == 0 {
 268                         assert!(bs_resp.2.is_none());
 269
 270                         assert!(as_resp.1.is_some());
 271                         assert!(as_resp.2.is_some());
 272                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 273                 } else {
 274                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 275                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 276                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 277                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 278                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 279                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 280
 281                         assert!(as_resp.1.is_none());
 282
 283                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 284                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 285                         assert_eq!(events_3.len(), 1);
 286                         match events_3[0] {
 287                                 Event::PaymentSent { ref payment_preimage } => {
 288                                         assert_eq!(*payment_preimage, payment_preimage_1);
 289                                 },
 290                                 _ => panic!("Unexpected event"),
 291                         }
 292
 293                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 294                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 295                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 296                         check_added_monitors!(nodes[0], 1);
 297
 298                         as_resp.1 = Some(as_resp_raa);
 299                         bs_resp.2 = None;
 300                 }
 301
 302                 if disconnect_count & !disconnect_flags > 1 {
 303                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 304
 305                         if (disconnect_count & 16) == 0 {
 306                                 assert!(reestablish_1 == second_reestablish_1);
 307                                 assert!(reestablish_2 == second_reestablish_2);
 308                         }
 309                         assert!(as_resp == second_as_resp);
 310                         assert!(bs_resp == second_bs_resp);
 311                 }
 312
 313                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 314         } else {
 315                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 316                 assert_eq!(events_4.len(), 2);
 317                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 318                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 319                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 320                                 msg.clone()
 321                         },
 322                         _ => panic!("Unexpected event"),
 323                 })
 324         };
 325
 326         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 327
 328         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 329         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 330         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 331         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 332         check_added_monitors!(nodes[1], 1);
 333
 334         if disconnect_count & !disconnect_flags > 2 {
 335                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 336
 337                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 338                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 339
 340                 assert!(as_resp.2.is_none());
 341                 assert!(bs_resp.2.is_none());
 342         }
 343
 344         let as_commitment_update;
 345         let bs_second_commitment_update;
 346
 347         macro_rules! handle_bs_raa { () => {
 348                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 349                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 350                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 351                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 352                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 353                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 354                 assert!(as_commitment_update.update_fee.is_none());
 355                 check_added_monitors!(nodes[0], 1);
 356         } }
 357
 358         macro_rules! handle_initial_raa { () => {
 359                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 360                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 361                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 362                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 363                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 364                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 365                 assert!(bs_second_commitment_update.update_fee.is_none());
 366                 check_added_monitors!(nodes[1], 1);
 367         } }
 368
 369         if (disconnect_count & 8) == 0 {
 370                 handle_bs_raa!();
 371
 372                 if disconnect_count & !disconnect_flags > 3 {
 373                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 374
 375                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 376                         assert!(bs_resp.1.is_none());
 377
 378                         assert!(as_resp.2.unwrap() == as_commitment_update);
 379                         assert!(bs_resp.2.is_none());
 380
 381                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 382                 }
 383
 384                 handle_initial_raa!();
 385
 386                 if disconnect_count & !disconnect_flags > 4 {
 387                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 388
 389                         assert!(as_resp.1.is_none());
 390                         assert!(bs_resp.1.is_none());
 391
 392                         assert!(as_resp.2.unwrap() == as_commitment_update);
 393                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 394                 }
 395         } else {
 396                 handle_initial_raa!();
 397
 398                 if disconnect_count & !disconnect_flags > 3 {
 399                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 400
 401                         assert!(as_resp.1.is_none());
 402                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 403
 404                         assert!(as_resp.2.is_none());
 405                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 406
 407                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 408                 }
 409
 410                 handle_bs_raa!();
 411
 412                 if disconnect_count & !disconnect_flags > 4 {
 413                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 414
 415                         assert!(as_resp.1.is_none());
 416                         assert!(bs_resp.1.is_none());
 417
 418                         assert!(as_resp.2.unwrap() == as_commitment_update);
 419                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 420                 }
 421         }
 422
 423         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 424         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 425         // No commitment_signed so get_event_msg's assert(len == 1) passes
 426         check_added_monitors!(nodes[0], 1);
 427
 428         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 429         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 430         // No commitment_signed so get_event_msg's assert(len == 1) passes
 431         check_added_monitors!(nodes[1], 1);
 432
 433         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 434         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 435         check_added_monitors!(nodes[1], 1);
 436
 437         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 438         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 439         check_added_monitors!(nodes[0], 1);
 440
 441         expect_pending_htlcs_forwardable!(nodes[1]);
 442
 443         let events_5 = nodes[1].node.get_and_clear_pending_events();
 444         assert_eq!(events_5.len(), 1);
 445         match events_5[0] {
 446                 Event::PaymentReceived { ref payment_hash, amt } => {
 447                         assert_eq!(payment_hash_2, *payment_hash);
 448                         assert_eq!(amt, 1000000);
 449                 },
 450                 _ => panic!("Unexpected event"),
 451         }
 452
 453         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 454 }
 455
 456 #[test]
 457 fn test_monitor_temporary_update_fail_a() {
 458         do_test_monitor_temporary_update_fail(0);
 459         do_test_monitor_temporary_update_fail(1);
 460         do_test_monitor_temporary_update_fail(2);
 461         do_test_monitor_temporary_update_fail(3);
 462         do_test_monitor_temporary_update_fail(4);
 463         do_test_monitor_temporary_update_fail(5);
 464 }
 465
 466 #[test]
 467 fn test_monitor_temporary_update_fail_b() {
 468         do_test_monitor_temporary_update_fail(2 | 8);
 469         do_test_monitor_temporary_update_fail(3 | 8);
 470         do_test_monitor_temporary_update_fail(4 | 8);
 471         do_test_monitor_temporary_update_fail(5 | 8);
 472 }
 473
 474 #[test]
 475 fn test_monitor_temporary_update_fail_c() {
 476         do_test_monitor_temporary_update_fail(1 | 16);
 477         do_test_monitor_temporary_update_fail(2 | 16);
 478         do_test_monitor_temporary_update_fail(3 | 16);
 479         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 480         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 481 }
 482
 483 #[test]
 484 fn test_monitor_update_fail_cs() {
 485         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 486         let chanmon_cfgs = create_chanmon_cfgs(2);
 487         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 488         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 489         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 490         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 491
 492         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 493         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 494         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 495         check_added_monitors!(nodes[0], 1);
 496
 497         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 498         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 499
 500         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 501         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 502         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 503         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 504         check_added_monitors!(nodes[1], 1);
 505         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 506
 507         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 508         nodes[1].node.test_restore_channel_monitor();
 509         check_added_monitors!(nodes[1], 1);
 510         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 511         assert_eq!(responses.len(), 2);
 512
 513         match responses[0] {
 514                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 515                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 516                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 517                         check_added_monitors!(nodes[0], 1);
 518                 },
 519                 _ => panic!("Unexpected event"),
 520         }
 521         match responses[1] {
 522                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 523                         assert!(updates.update_add_htlcs.is_empty());
 524                         assert!(updates.update_fulfill_htlcs.is_empty());
 525                         assert!(updates.update_fail_htlcs.is_empty());
 526                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 527                         assert!(updates.update_fee.is_none());
 528                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 529
 530                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 531                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 532                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 533                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 534                         check_added_monitors!(nodes[0], 1);
 535                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 536                 },
 537                 _ => panic!("Unexpected event"),
 538         }
 539
 540         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 541         nodes[0].node.test_restore_channel_monitor();
 542         check_added_monitors!(nodes[0], 1);
 543
 544         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 545         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 546         check_added_monitors!(nodes[1], 1);
 547
 548         expect_pending_htlcs_forwardable!(nodes[1]);
 549
 550         let events = nodes[1].node.get_and_clear_pending_events();
 551         assert_eq!(events.len(), 1);
 552         match events[0] {
 553                 Event::PaymentReceived { payment_hash, amt } => {
 554                         assert_eq!(payment_hash, our_payment_hash);
 555                         assert_eq!(amt, 1000000);
 556                 },
 557                 _ => panic!("Unexpected event"),
 558         };
 559
 560         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 561 }
 562
 563 #[test]
 564 fn test_monitor_update_fail_no_rebroadcast() {
 565         // Tests handling of a monitor update failure when no message rebroadcasting on
 566         // test_restore_channel_monitor() is required. Backported from
 567         // chanmon_fail_consistency fuzz tests.
 568         let chanmon_cfgs = create_chanmon_cfgs(2);
 569         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 570         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 571         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 572         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 573
 574         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 575         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 576         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 577         check_added_monitors!(nodes[0], 1);
 578
 579         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 580         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 581         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 582
 583         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 584         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 585         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 586         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 587         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 588         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 589         check_added_monitors!(nodes[1], 1);
 590
 591         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 592         nodes[1].node.test_restore_channel_monitor();
 593         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 594         check_added_monitors!(nodes[1], 1);
 595         expect_pending_htlcs_forwardable!(nodes[1]);
 596
 597         let events = nodes[1].node.get_and_clear_pending_events();
 598         assert_eq!(events.len(), 1);
 599         match events[0] {
 600                 Event::PaymentReceived { payment_hash, .. } => {
 601                         assert_eq!(payment_hash, our_payment_hash);
 602                 },
 603                 _ => panic!("Unexpected event"),
 604         }
 605
 606         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 607 }
 608
 609 #[test]
 610 fn test_monitor_update_raa_while_paused() {
 611         // Tests handling of an RAA while monitor updating has already been marked failed.
 612         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 613         let chanmon_cfgs = create_chanmon_cfgs(2);
 614         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 615         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 616         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 617         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 618
 619         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 620
 621         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 622         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 623         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 624         check_added_monitors!(nodes[0], 1);
 625         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 626
 627         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 628         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 629         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 630         check_added_monitors!(nodes[1], 1);
 631         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 632
 633         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 634         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 635         check_added_monitors!(nodes[1], 1);
 636         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 637
 638         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 639         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 640         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 641         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 642         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 643         check_added_monitors!(nodes[0], 1);
 644
 645         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 646         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 647         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 648         check_added_monitors!(nodes[0], 1);
 649
 650         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 651         nodes[0].node.test_restore_channel_monitor();
 652         check_added_monitors!(nodes[0], 1);
 653
 654         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 655         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 656         check_added_monitors!(nodes[1], 1);
 657         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 658
 659         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 660         check_added_monitors!(nodes[1], 1);
 661         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 662
 663         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 664         check_added_monitors!(nodes[0], 1);
 665         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 666
 667         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 668         check_added_monitors!(nodes[0], 1);
 669         expect_pending_htlcs_forwardable!(nodes[0]);
 670         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 671
 672         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 673         check_added_monitors!(nodes[1], 1);
 674         expect_pending_htlcs_forwardable!(nodes[1]);
 675         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 676
 677         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 678         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 679 }
 680
 681 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 682         // Tests handling of a monitor update failure when processing an incoming RAA
 683         let chanmon_cfgs = create_chanmon_cfgs(3);
 684         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 685         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 686         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 687         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 688         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 689
 690         // Rebalance a bit so that we can send backwards from 2 to 1.
 691         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 692
 693         // Route a first payment that we'll fail backwards
 694         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 695
 696         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 697         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 698         expect_pending_htlcs_forwardable!(nodes[2]);
 699         check_added_monitors!(nodes[2], 1);
 700
 701         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 702         assert!(updates.update_add_htlcs.is_empty());
 703         assert!(updates.update_fulfill_htlcs.is_empty());
 704         assert_eq!(updates.update_fail_htlcs.len(), 1);
 705         assert!(updates.update_fail_malformed_htlcs.is_empty());
 706         assert!(updates.update_fee.is_none());
 707         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 708
 709         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 710         check_added_monitors!(nodes[0], 0);
 711
 712         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 713         // holding cell.
 714         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 715         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 716         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 717         check_added_monitors!(nodes[0], 1);
 718
 719         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 720         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 721         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 722
 723         expect_pending_htlcs_forwardable!(nodes[1]);
 724         check_added_monitors!(nodes[1], 0);
 725         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 726
 727         // Now fail monitor updating.
 728         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 729         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 730         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 731         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 732         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 733         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 734         check_added_monitors!(nodes[1], 1);
 735
 736         // Attempt to forward a third payment but fail due to the second channel being unavailable
 737         // for forwarding.
 738
 739         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 740         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 741         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 742         check_added_monitors!(nodes[0], 1);
 743
 744         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 745         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 746         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 747         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 748         check_added_monitors!(nodes[1], 0);
 749
 750         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 751         assert_eq!(events_2.len(), 1);
 752         match events_2.remove(0) {
 753                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 754                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 755                         assert!(updates.update_fulfill_htlcs.is_empty());
 756                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 757                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 758                         assert!(updates.update_add_htlcs.is_empty());
 759                         assert!(updates.update_fee.is_none());
 760
 761                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 762                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 763
 764                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 765                         assert_eq!(msg_events.len(), 1);
 766                         match msg_events[0] {
 767                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 768                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 769                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 770                                 },
 771                                 _ => panic!("Unexpected event"),
 772                         }
 773
 774                         let events = nodes[0].node.get_and_clear_pending_events();
 775                         assert_eq!(events.len(), 1);
 776                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 777                                 assert_eq!(payment_hash, payment_hash_3);
 778                                 assert!(!rejected_by_dest);
 779                         } else { panic!("Unexpected event!"); }
 780                 },
 781                 _ => panic!("Unexpected event type!"),
 782         };
 783
 784         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 785                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 786                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 787                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 788                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 789                 check_added_monitors!(nodes[2], 1);
 790
 791                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 792                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 793                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 794                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 795                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 796                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 797                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 798                 (Some(payment_preimage_4), Some(payment_hash_4))
 799         } else { (None, None) };
 800
 801         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 802         // update_add update.
 803         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 804         nodes[1].node.test_restore_channel_monitor();
 805         check_added_monitors!(nodes[1], 1);
 806         expect_pending_htlcs_forwardable!(nodes[1]);
 807         check_added_monitors!(nodes[1], 1);
 808
 809         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 810         if test_ignore_second_cs {
 811                 assert_eq!(events_3.len(), 3);
 812         } else {
 813                 assert_eq!(events_3.len(), 2);
 814         }
 815
 816         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 817         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 818         let messages_a = match events_3.pop().unwrap() {
 819                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 820                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 821                         assert!(updates.update_fulfill_htlcs.is_empty());
 822                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 823                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 824                         assert!(updates.update_add_htlcs.is_empty());
 825                         assert!(updates.update_fee.is_none());
 826                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 827                 },
 828                 _ => panic!("Unexpected event type!"),
 829         };
 830         let raa = if test_ignore_second_cs {
 831                 match events_3.remove(1) {
 832                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 833                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 834                                 Some(msg.clone())
 835                         },
 836                         _ => panic!("Unexpected event"),
 837                 }
 838         } else { None };
 839         let send_event_b = SendEvent::from_event(events_3.remove(0));
 840         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 841
 842         // Now deliver the new messages...
 843
 844         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 845         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 846         let events_4 = nodes[0].node.get_and_clear_pending_events();
 847         assert_eq!(events_4.len(), 1);
 848         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 849                 assert_eq!(payment_hash, payment_hash_1);
 850                 assert!(rejected_by_dest);
 851         } else { panic!("Unexpected event!"); }
 852
 853         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 854         if test_ignore_second_cs {
 855                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 856                 check_added_monitors!(nodes[2], 1);
 857                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 858                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 859                 check_added_monitors!(nodes[2], 1);
 860                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 861                 assert!(bs_cs.update_add_htlcs.is_empty());
 862                 assert!(bs_cs.update_fail_htlcs.is_empty());
 863                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 864                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 865                 assert!(bs_cs.update_fee.is_none());
 866
 867                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 868                 check_added_monitors!(nodes[1], 1);
 869                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 870                 assert!(as_cs.update_add_htlcs.is_empty());
 871                 assert!(as_cs.update_fail_htlcs.is_empty());
 872                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 873                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 874                 assert!(as_cs.update_fee.is_none());
 875
 876                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 877                 check_added_monitors!(nodes[1], 1);
 878                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 879
 880                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 881                 check_added_monitors!(nodes[2], 1);
 882                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 883
 884                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 885                 check_added_monitors!(nodes[2], 1);
 886                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 887
 888                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 889                 check_added_monitors!(nodes[1], 1);
 890                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 891         } else {
 892                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 893         }
 894
 895         expect_pending_htlcs_forwardable!(nodes[2]);
 896
 897         let events_6 = nodes[2].node.get_and_clear_pending_events();
 898         assert_eq!(events_6.len(), 1);
 899         match events_6[0] {
 900                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 901                 _ => panic!("Unexpected event"),
 902         };
 903
 904         if test_ignore_second_cs {
 905                 expect_pending_htlcs_forwardable!(nodes[1]);
 906                 check_added_monitors!(nodes[1], 1);
 907
 908                 send_event = SendEvent::from_node(&nodes[1]);
 909                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 910                 assert_eq!(send_event.msgs.len(), 1);
 911                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 912                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 913
 914                 expect_pending_htlcs_forwardable!(nodes[0]);
 915
 916                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 917                 assert_eq!(events_9.len(), 1);
 918                 match events_9[0] {
 919                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 920                         _ => panic!("Unexpected event"),
 921                 };
 922                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 923         }
 924
 925         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 926 }
 927
 928 #[test]
 929 fn test_monitor_update_fail_raa() {
 930         do_test_monitor_update_fail_raa(false);
 931         do_test_monitor_update_fail_raa(true);
 932 }
 933
 934 #[test]
 935 fn test_monitor_update_fail_reestablish() {
 936         // Simple test for message retransmission after monitor update failure on
 937         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 938         // HTLCs).
 939         let chanmon_cfgs = create_chanmon_cfgs(3);
 940         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 941         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 942         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 943         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 944         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 945
 946         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 947
 948         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 949         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 950
 951         assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
 952         check_added_monitors!(nodes[2], 1);
 953         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 954         assert!(updates.update_add_htlcs.is_empty());
 955         assert!(updates.update_fail_htlcs.is_empty());
 956         assert!(updates.update_fail_malformed_htlcs.is_empty());
 957         assert!(updates.update_fee.is_none());
 958         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 959         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 960         check_added_monitors!(nodes[1], 1);
 961         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 962         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 963
 964         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 965         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 966         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 967
 968         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 969         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 970
 971         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 972
 973         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 974         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 975         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 976         check_added_monitors!(nodes[1], 1);
 977
 978         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 979         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 980
 981         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 982         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 983
 984         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 985         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 986
 987         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 988
 989         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 990         check_added_monitors!(nodes[1], 0);
 991         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 992
 993         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 994         nodes[1].node.test_restore_channel_monitor();
 995         check_added_monitors!(nodes[1], 1);
 996
 997         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 998         assert!(updates.update_add_htlcs.is_empty());
 999         assert!(updates.update_fail_htlcs.is_empty());
1000         assert!(updates.update_fail_malformed_htlcs.is_empty());
1001         assert!(updates.update_fee.is_none());
1002         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1003         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1004         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1005
1006         let events = nodes[0].node.get_and_clear_pending_events();
1007         assert_eq!(events.len(), 1);
1008         match events[0] {
1009                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1010                 _ => panic!("Unexpected event"),
1011         }
1012 }
1013
1014 #[test]
1015 fn raa_no_response_awaiting_raa_state() {
1016         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1017         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1018         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1019         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1020         let chanmon_cfgs = create_chanmon_cfgs(2);
1021         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1022         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1023         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1024         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1025
1026         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1027         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1028         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1029         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1030
1031         // Queue up two payments - one will be delivered right away, one immediately goes into the
1032         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1033         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1034         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1035         // generation during RAA while in monitor-update-failed state.
1036         nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1037         check_added_monitors!(nodes[0], 1);
1038         nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1039         check_added_monitors!(nodes[0], 0);
1040
1041         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1042         assert_eq!(events.len(), 1);
1043         let payment_event = SendEvent::from_event(events.pop().unwrap());
1044         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1045         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1046         check_added_monitors!(nodes[1], 1);
1047
1048         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1049         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1050         check_added_monitors!(nodes[0], 1);
1051         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1052         assert_eq!(events.len(), 1);
1053         let payment_event = SendEvent::from_event(events.pop().unwrap());
1054
1055         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1056         check_added_monitors!(nodes[0], 1);
1057         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1058
1059         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1060         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1061         // then restore channel monitor updates.
1062         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1063         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1064         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1065         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1066         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1067         check_added_monitors!(nodes[1], 1);
1068
1069         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1070         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1071         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1072         check_added_monitors!(nodes[1], 1);
1073
1074         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1075         nodes[1].node.test_restore_channel_monitor();
1076         // nodes[1] should be AwaitingRAA here!
1077         check_added_monitors!(nodes[1], 1);
1078         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1079         expect_pending_htlcs_forwardable!(nodes[1]);
1080         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1081
1082         // We send a third payment here, which is somewhat of a redundant test, but the
1083         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1084         // commitment transaction states) whereas here we can explicitly check for it.
1085         nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1086         check_added_monitors!(nodes[0], 0);
1087         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1088
1089         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1090         check_added_monitors!(nodes[0], 1);
1091         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1092         assert_eq!(events.len(), 1);
1093         let payment_event = SendEvent::from_event(events.pop().unwrap());
1094
1095         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1096         check_added_monitors!(nodes[0], 1);
1097         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1098
1099         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1100         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1101         check_added_monitors!(nodes[1], 1);
1102         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1103
1104         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1105         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1106         check_added_monitors!(nodes[1], 1);
1107         expect_pending_htlcs_forwardable!(nodes[1]);
1108         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1109         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1110
1111         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1112         check_added_monitors!(nodes[0], 1);
1113
1114         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1115         check_added_monitors!(nodes[0], 1);
1116         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1117
1118         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1119         check_added_monitors!(nodes[1], 1);
1120         expect_pending_htlcs_forwardable!(nodes[1]);
1121         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1122
1123         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1124         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1125         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1126 }
1127
1128 #[test]
1129 fn claim_while_disconnected_monitor_update_fail() {
1130         // Test for claiming a payment while disconnected and then having the resulting
1131         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1132         // contrived case for nodes with network instability.
1133         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1134         // code introduced a regression in this test (specifically, this caught a removal of the
1135         // channel_reestablish handling ensuring the order was sensical given the messages used).
1136         let chanmon_cfgs = create_chanmon_cfgs(2);
1137         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1138         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1139         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1140         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1141
1142         // Forward a payment for B to claim
1143         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1144
1145         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1146         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1147
1148         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1149         check_added_monitors!(nodes[1], 1);
1150
1151         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1152         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1153
1154         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1155         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1156
1157         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1158         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1159
1160         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1161         // update.
1162         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1163
1164         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1165         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1166         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1167         check_added_monitors!(nodes[1], 1);
1168         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1169
1170         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1171         // the monitor still failed
1172         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1173         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1174         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1175         check_added_monitors!(nodes[0], 1);
1176
1177         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1178         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1179         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1180         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1181         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1182         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1183         // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1184
1185         // Now un-fail the monitor, which will result in B sending its original commitment update,
1186         // receiving the commitment update from A, and the resulting commitment dances.
1187         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1188         nodes[1].node.test_restore_channel_monitor();
1189         check_added_monitors!(nodes[1], 1);
1190
1191         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1192         assert_eq!(bs_msgs.len(), 2);
1193
1194         match bs_msgs[0] {
1195                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1196                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1197                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1198                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1199                         check_added_monitors!(nodes[0], 1);
1200
1201                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1202                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1203                         check_added_monitors!(nodes[1], 1);
1204                 },
1205                 _ => panic!("Unexpected event"),
1206         }
1207
1208         match bs_msgs[1] {
1209                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1210                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1211                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1212                         check_added_monitors!(nodes[0], 1);
1213                 },
1214                 _ => panic!("Unexpected event"),
1215         }
1216
1217         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1218
1219         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1220         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1221         check_added_monitors!(nodes[0], 1);
1222         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1223
1224         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1225         check_added_monitors!(nodes[1], 1);
1226         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1227         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1228         check_added_monitors!(nodes[1], 1);
1229
1230         expect_pending_htlcs_forwardable!(nodes[1]);
1231         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1232
1233         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1234         check_added_monitors!(nodes[0], 1);
1235
1236         let events = nodes[0].node.get_and_clear_pending_events();
1237         assert_eq!(events.len(), 1);
1238         match events[0] {
1239                 Event::PaymentSent { ref payment_preimage } => {
1240                         assert_eq!(*payment_preimage, payment_preimage_1);
1241                 },
1242                 _ => panic!("Unexpected event"),
1243         }
1244
1245         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1246 }
1247
1248 #[test]
1249 fn monitor_failed_no_reestablish_response() {
1250         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1251         // response to a commitment_signed.
1252         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1253         // debug_assert!() failure in channel_reestablish handling.
1254         let chanmon_cfgs = create_chanmon_cfgs(2);
1255         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1256         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1257         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1258         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1259
1260         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1261         // on receipt).
1262         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1263         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1264         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1265         check_added_monitors!(nodes[0], 1);
1266
1267         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1268         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1269         assert_eq!(events.len(), 1);
1270         let payment_event = SendEvent::from_event(events.pop().unwrap());
1271         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1272         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1273         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1274         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1275         check_added_monitors!(nodes[1], 1);
1276
1277         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1278         // is still failing to update monitors.
1279         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1280         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1281
1282         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1283         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1284
1285         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1286         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1287
1288         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1289         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1290
1291         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1292         nodes[1].node.test_restore_channel_monitor();
1293         check_added_monitors!(nodes[1], 1);
1294         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1295
1296         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1297         check_added_monitors!(nodes[0], 1);
1298         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1299         check_added_monitors!(nodes[0], 1);
1300
1301         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1302         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1303         check_added_monitors!(nodes[1], 1);
1304
1305         expect_pending_htlcs_forwardable!(nodes[1]);
1306         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1307
1308         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1309 }
1310
1311 #[test]
1312 fn first_message_on_recv_ordering() {
1313         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1314         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1315         // a commitment_signed which needs to send an RAA first.
1316         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1317         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1318         // response. To do this, we start routing two payments, with the final RAA for the first being
1319         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1320         // have no pending response but will want to send a RAA/CS (with the updates for the second
1321         // payment applied).
1322         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1323         let chanmon_cfgs = create_chanmon_cfgs(2);
1324         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1325         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1326         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1327         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1328
1329         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1330         // can deliver it and fail the monitor update.
1331         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1332         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1333         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1334         check_added_monitors!(nodes[0], 1);
1335
1336         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1337         assert_eq!(events.len(), 1);
1338         let payment_event = SendEvent::from_event(events.pop().unwrap());
1339         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1340         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1341         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1342         check_added_monitors!(nodes[1], 1);
1343         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1344
1345         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1346         check_added_monitors!(nodes[0], 1);
1347         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1348         check_added_monitors!(nodes[0], 1);
1349
1350         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1351
1352         // Route the second payment, generating an update_add_htlc/commitment_signed
1353         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1354         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1355         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1356         check_added_monitors!(nodes[0], 1);
1357         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1358         assert_eq!(events.len(), 1);
1359         let payment_event = SendEvent::from_event(events.pop().unwrap());
1360         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1361
1362         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1363
1364         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1365         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1366         // to the next message also tests resetting the delivery order.
1367         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1368         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1369         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1370         check_added_monitors!(nodes[1], 1);
1371
1372         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1373         // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1374         // the appropriate HTLC acceptance).
1375         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1376         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1377         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1378         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1379
1380         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1381         nodes[1].node.test_restore_channel_monitor();
1382         check_added_monitors!(nodes[1], 1);
1383
1384         expect_pending_htlcs_forwardable!(nodes[1]);
1385         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1386
1387         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1388         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1389         check_added_monitors!(nodes[0], 1);
1390         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1391         check_added_monitors!(nodes[0], 1);
1392
1393         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1394         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1395         check_added_monitors!(nodes[1], 1);
1396
1397         expect_pending_htlcs_forwardable!(nodes[1]);
1398         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1399
1400         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1401         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1402 }
1403
1404 #[test]
1405 fn test_monitor_update_fail_claim() {
1406         // Basic test for monitor update failures when processing claim_funds calls.
1407         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1408         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1409         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1410         // updating and claim the payment on B.
1411         let chanmon_cfgs = create_chanmon_cfgs(3);
1412         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1413         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1414         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1415         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1416         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1417
1418         // Rebalance a bit so that we can send backwards from 3 to 2.
1419         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1420
1421         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1422
1423         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1424         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1425         check_added_monitors!(nodes[1], 1);
1426
1427         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1428         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1429         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1430         check_added_monitors!(nodes[2], 1);
1431
1432         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1433         // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1434         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1435
1436         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1437         assert_eq!(events.len(), 1);
1438         let payment_event = SendEvent::from_event(events.pop().unwrap());
1439         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1440         let events = nodes[1].node.get_and_clear_pending_msg_events();
1441         assert_eq!(events.len(), 0);
1442         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1443         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1444
1445         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1446         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1447         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1448
1449         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1450         assert_eq!(msg_events.len(), 1);
1451         match msg_events[0] {
1452                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1453                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1454                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1455                 },
1456                 _ => panic!("Unexpected event"),
1457         }
1458
1459         let events = nodes[2].node.get_and_clear_pending_events();
1460         assert_eq!(events.len(), 1);
1461         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1462                 assert_eq!(payment_hash, payment_hash_2);
1463                 assert!(!rejected_by_dest);
1464         } else { panic!("Unexpected event!"); }
1465
1466         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1467         nodes[1].node.test_restore_channel_monitor();
1468         check_added_monitors!(nodes[1], 1);
1469
1470         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1471         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1472         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1473
1474         let events = nodes[0].node.get_and_clear_pending_events();
1475         assert_eq!(events.len(), 1);
1476         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1477                 assert_eq!(payment_preimage, payment_preimage_1);
1478         } else { panic!("Unexpected event!"); }
1479 }
1480
1481 #[test]
1482 fn test_monitor_update_on_pending_forwards() {
1483         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1484         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1485         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1486         // from C to A will be pending a forward to A.
1487         let chanmon_cfgs = create_chanmon_cfgs(3);
1488         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1489         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1490         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1491         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1492         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1493
1494         // Rebalance a bit so that we can send backwards from 3 to 1.
1495         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1496
1497         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1498         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1499         expect_pending_htlcs_forwardable!(nodes[2]);
1500         check_added_monitors!(nodes[2], 1);
1501
1502         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1503         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1504         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1505         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1506
1507         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1508         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1509         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1510         check_added_monitors!(nodes[2], 1);
1511
1512         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1513         assert_eq!(events.len(), 1);
1514         let payment_event = SendEvent::from_event(events.pop().unwrap());
1515         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1516         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1517
1518         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1519         expect_pending_htlcs_forwardable!(nodes[1]);
1520         check_added_monitors!(nodes[1], 1);
1521         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1522         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1523
1524         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1525         nodes[1].node.test_restore_channel_monitor();
1526         check_added_monitors!(nodes[1], 1);
1527
1528         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1529         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1530         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1531         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1532
1533         let events = nodes[0].node.get_and_clear_pending_events();
1534         assert_eq!(events.len(), 2);
1535         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1536                 assert_eq!(payment_hash, payment_hash_1);
1537                 assert!(rejected_by_dest);
1538         } else { panic!("Unexpected event!"); }
1539         match events[1] {
1540                 Event::PendingHTLCsForwardable { .. } => { },
1541                 _ => panic!("Unexpected event"),
1542         };
1543         nodes[0].node.process_pending_htlc_forwards();
1544         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1545
1546         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1547 }
1548
1549 #[test]
1550 fn monitor_update_claim_fail_no_response() {
1551         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1552         // to channel being AwaitingRAA).
1553         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1554         // code was broken.
1555         let chanmon_cfgs = create_chanmon_cfgs(2);
1556         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1557         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1558         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1559         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1560
1561         // Forward a payment for B to claim
1562         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1563
1564         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1565         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1566         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1567         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1568         check_added_monitors!(nodes[0], 1);
1569
1570         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1571         assert_eq!(events.len(), 1);
1572         let payment_event = SendEvent::from_event(events.pop().unwrap());
1573         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1574         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1575
1576         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1577         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1578         check_added_monitors!(nodes[1], 1);
1579         let events = nodes[1].node.get_and_clear_pending_msg_events();
1580         assert_eq!(events.len(), 0);
1581         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1582
1583         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1584         nodes[1].node.test_restore_channel_monitor();
1585         check_added_monitors!(nodes[1], 1);
1586         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1587
1588         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1589         check_added_monitors!(nodes[1], 1);
1590         expect_pending_htlcs_forwardable!(nodes[1]);
1591         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1592
1593         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1594         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1595         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1596
1597         let events = nodes[0].node.get_and_clear_pending_events();
1598         assert_eq!(events.len(), 1);
1599         match events[0] {
1600                 Event::PaymentSent { ref payment_preimage } => {
1601                         assert_eq!(*payment_preimage, payment_preimage_1);
1602                 },
1603                 _ => panic!("Unexpected event"),
1604         }
1605
1606         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1607 }
1608
1609 // Note that restore_between_fails with !fail_on_generate is useless
1610 // Also note that !fail_on_generate && !fail_on_signed is useless
1611 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1612 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1613 // restore_b_before_conf has no meaning if !confirm_a_first
1614 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1615         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1616         // the channel setup happily after the update is restored.
1617         let chanmon_cfgs = create_chanmon_cfgs(2);
1618         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1619         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1620         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1621
1622         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1623         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1624         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1625
1626         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1627
1628         if fail_on_generate {
1629                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1630         }
1631         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1632         check_added_monitors!(nodes[0], 1);
1633
1634         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1635         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1636         check_added_monitors!(nodes[1], 1);
1637
1638         if restore_between_fails {
1639                 assert!(fail_on_generate);
1640                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1641                 nodes[0].node.test_restore_channel_monitor();
1642                 check_added_monitors!(nodes[0], 1);
1643                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1644                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1645         }
1646
1647         if fail_on_signed {
1648                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1649         } else {
1650                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1651                 assert!(fail_on_generate); // Somebody has to fail
1652         }
1653         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1654         if fail_on_signed || !restore_between_fails {
1655                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1656                 if fail_on_generate && !restore_between_fails {
1657                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1658                         check_added_monitors!(nodes[0], 0);
1659                 } else {
1660                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1661                         check_added_monitors!(nodes[0], 1);
1662                 }
1663                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1664                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1665                 nodes[0].node.test_restore_channel_monitor();
1666         }
1667
1668         check_added_monitors!(nodes[0], 1);
1669
1670         let events = nodes[0].node.get_and_clear_pending_events();
1671         assert_eq!(events.len(), 1);
1672         match events[0] {
1673                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1674                         assert_eq!(user_channel_id, 43);
1675                         assert_eq!(*funding_txo, funding_output);
1676                 },
1677                 _ => panic!("Unexpected event"),
1678         };
1679
1680         if confirm_a_first {
1681                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1682                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1683         } else {
1684                 assert!(!restore_b_before_conf);
1685                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1686                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1687         }
1688
1689         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1690         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1691         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1692         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1693         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1694         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1695
1696         if !restore_b_before_conf {
1697                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1698                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1699                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1700         }
1701
1702         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1703         nodes[1].node.test_restore_channel_monitor();
1704         check_added_monitors!(nodes[1], 1);
1705
1706         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1707                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1708
1709                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1710                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1711                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1712         } else {
1713                 if restore_b_before_conf {
1714                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1715                 }
1716                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1717                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1718         };
1719         for node in nodes.iter() {
1720                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1721                 node.router.handle_channel_update(&as_update).unwrap();
1722                 node.router.handle_channel_update(&bs_update).unwrap();
1723         }
1724
1725         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1726         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1727 }
1728
1729 #[test]
1730 fn during_funding_monitor_fail() {
1731         do_during_funding_monitor_fail(false, false, true, true, true);
1732         do_during_funding_monitor_fail(true, false, true, false, false);
1733         do_during_funding_monitor_fail(true, true, true, true, false);
1734         do_during_funding_monitor_fail(true, true, false, false, false);
1735 }