_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use chain::transaction::OutPoint;
   7 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure};
   8 use ln::channelmonitor::ChannelMonitorUpdateErr;
   9 use ln::features::InitFeatures;
  10 use ln::msgs;
  11 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  12 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  13 use util::errors::APIError;
  14
  15 use bitcoin::hashes::sha256::Hash as Sha256;
  16 use bitcoin::hashes::Hash;
  17
  18 use ln::functional_test_utils::*;
  19
  20 #[test]
  21 fn test_simple_monitor_permanent_update_fail() {
  22         // Test that we handle a simple permanent monitor update failure
  23         let chanmon_cfgs = create_chanmon_cfgs(2);
  24         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  25         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  26         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  27         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
  28
  29         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  30         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  31
  32         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  33         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), true, APIError::ChannelUnavailable {..}, {});
  34         check_added_monitors!(nodes[0], 2);
  35
  36         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  37         assert_eq!(events_1.len(), 2);
  38         match events_1[0] {
  39                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  40                 _ => panic!("Unexpected event"),
  41         };
  42         match events_1[1] {
  43                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  44                 _ => panic!("Unexpected event"),
  45         };
  46
  47         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  48         // PaymentFailed event
  49
  50         assert_eq!(nodes[0].node.list_channels().len(), 0);
  51 }
  52
  53 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  54         // Test that we can recover from a simple temporary monitor update failure optionally with
  55         // a disconnect in between
  56         let chanmon_cfgs = create_chanmon_cfgs(2);
  57         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  58         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  59         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  60         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
  61
  62         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  63         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  64
  65         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  66
  67         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), false, APIError::MonitorUpdateFailed, {});
  68         check_added_monitors!(nodes[0], 1);
  69
  70         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  71         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  72         assert_eq!(nodes[0].node.list_channels().len(), 1);
  73
  74         if disconnect {
  75                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  76                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  77                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  78         }
  79
  80         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  81         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
  82         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
  83         check_added_monitors!(nodes[0], 0);
  84
  85         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  86         assert_eq!(events_2.len(), 1);
  87         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  88         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  89         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  90         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  91
  92         expect_pending_htlcs_forwardable!(nodes[1]);
  93
  94         let events_3 = nodes[1].node.get_and_clear_pending_events();
  95         assert_eq!(events_3.len(), 1);
  96         match events_3[0] {
  97                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
  98                         assert_eq!(payment_hash_1, *payment_hash);
  99                         assert_eq!(*payment_secret, None);
 100                         assert_eq!(amt, 1000000);
 101                 },
 102                 _ => panic!("Unexpected event"),
 103         }
 104
 105         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 106
 107         // Now set it to failed again...
 108         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 109         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 110         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
 111         check_added_monitors!(nodes[0], 1);
 112
 113         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 114         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 115         assert_eq!(nodes[0].node.list_channels().len(), 1);
 116
 117         if disconnect {
 118                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 119                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 120                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 121         }
 122
 123         // ...and make sure we can force-close a frozen channel
 124         nodes[0].node.force_close_channel(&channel_id);
 125         check_added_monitors!(nodes[0], 1);
 126         check_closed_broadcast!(nodes[0], false);
 127
 128         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 129         // PaymentFailed event
 130
 131         assert_eq!(nodes[0].node.list_channels().len(), 0);
 132 }
 133
 134 #[test]
 135 fn test_simple_monitor_temporary_update_fail() {
 136         do_test_simple_monitor_temporary_update_fail(false);
 137         do_test_simple_monitor_temporary_update_fail(true);
 138 }
 139
 140 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 141         let disconnect_flags = 8 | 16;
 142
 143         // Test that we can recover from a temporary monitor update failure with some in-flight
 144         // HTLCs going on at the same time potentially with some disconnection thrown in.
 145         // * First we route a payment, then get a temporary monitor update failure when trying to
 146         //   route a second payment. We then claim the first payment.
 147         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 148         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 149         //   the ChannelMonitor on a watchtower).
 150         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 151         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 152         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 153         //   disconnect_count & !disconnect_flags is 0).
 154         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 155         //   through message sending, potentially disconnect/reconnecting multiple times based on
 156         //   disconnect_count, to get the update_fulfill_htlc through.
 157         // * We then walk through more message exchanges to get the original update_add_htlc
 158         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 159         //   disconnect/reconnecting based on disconnect_count.
 160         let chanmon_cfgs = create_chanmon_cfgs(2);
 161         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 162         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 163         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 164         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 165
 166         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 167
 168         // Now try to send a second payment which will fail to send
 169         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 170         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 171
 172         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 173         unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
 174         check_added_monitors!(nodes[0], 1);
 175
 176         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 177         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 178         assert_eq!(nodes[0].node.list_channels().len(), 1);
 179
 180         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 181         // but nodes[0] won't respond since it is frozen.
 182         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
 183         check_added_monitors!(nodes[1], 1);
 184         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 185         assert_eq!(events_2.len(), 1);
 186         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 187                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 188                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 189                         assert!(update_add_htlcs.is_empty());
 190                         assert_eq!(update_fulfill_htlcs.len(), 1);
 191                         assert!(update_fail_htlcs.is_empty());
 192                         assert!(update_fail_malformed_htlcs.is_empty());
 193                         assert!(update_fee.is_none());
 194
 195                         if (disconnect_count & 16) == 0 {
 196                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 197                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 198                                 assert_eq!(events_3.len(), 1);
 199                                 match events_3[0] {
 200                                         Event::PaymentSent { ref payment_preimage } => {
 201                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 202                                         },
 203                                         _ => panic!("Unexpected event"),
 204                                 }
 205
 206                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 207                                 check_added_monitors!(nodes[0], 1);
 208                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 209                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 210                         }
 211
 212                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 213                 },
 214                 _ => panic!("Unexpected event"),
 215         };
 216
 217         if disconnect_count & !disconnect_flags > 0 {
 218                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 219                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 220         }
 221
 222         // Now fix monitor updating...
 223         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 224         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 225         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 226         check_added_monitors!(nodes[0], 0);
 227
 228         macro_rules! disconnect_reconnect_peers { () => { {
 229                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 230                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 231
 232                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 233                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 234                 assert_eq!(reestablish_1.len(), 1);
 235                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 236                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 237                 assert_eq!(reestablish_2.len(), 1);
 238
 239                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 240                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 241                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 242                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 243
 244                 assert!(as_resp.0.is_none());
 245                 assert!(bs_resp.0.is_none());
 246
 247                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 248         } } }
 249
 250         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 251                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 252                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 253
 254                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 255                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 256                 assert_eq!(reestablish_1.len(), 1);
 257                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 258                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 259                 assert_eq!(reestablish_2.len(), 1);
 260
 261                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 262                 check_added_monitors!(nodes[0], 0);
 263                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 264                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 265                 check_added_monitors!(nodes[1], 0);
 266                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 267
 268                 assert!(as_resp.0.is_none());
 269                 assert!(bs_resp.0.is_none());
 270
 271                 assert!(bs_resp.1.is_none());
 272                 if (disconnect_count & 16) == 0 {
 273                         assert!(bs_resp.2.is_none());
 274
 275                         assert!(as_resp.1.is_some());
 276                         assert!(as_resp.2.is_some());
 277                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 278                 } else {
 279                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 280                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 281                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 282                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 283                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 284                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 285
 286                         assert!(as_resp.1.is_none());
 287
 288                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 289                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 290                         assert_eq!(events_3.len(), 1);
 291                         match events_3[0] {
 292                                 Event::PaymentSent { ref payment_preimage } => {
 293                                         assert_eq!(*payment_preimage, payment_preimage_1);
 294                                 },
 295                                 _ => panic!("Unexpected event"),
 296                         }
 297
 298                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 299                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 300                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 301                         check_added_monitors!(nodes[0], 1);
 302
 303                         as_resp.1 = Some(as_resp_raa);
 304                         bs_resp.2 = None;
 305                 }
 306
 307                 if disconnect_count & !disconnect_flags > 1 {
 308                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 309
 310                         if (disconnect_count & 16) == 0 {
 311                                 assert!(reestablish_1 == second_reestablish_1);
 312                                 assert!(reestablish_2 == second_reestablish_2);
 313                         }
 314                         assert!(as_resp == second_as_resp);
 315                         assert!(bs_resp == second_bs_resp);
 316                 }
 317
 318                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 319         } else {
 320                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 321                 assert_eq!(events_4.len(), 2);
 322                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 323                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 324                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 325                                 msg.clone()
 326                         },
 327                         _ => panic!("Unexpected event"),
 328                 })
 329         };
 330
 331         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 332
 333         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 334         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 335         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 336         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 337         check_added_monitors!(nodes[1], 1);
 338
 339         if disconnect_count & !disconnect_flags > 2 {
 340                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 341
 342                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 343                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 344
 345                 assert!(as_resp.2.is_none());
 346                 assert!(bs_resp.2.is_none());
 347         }
 348
 349         let as_commitment_update;
 350         let bs_second_commitment_update;
 351
 352         macro_rules! handle_bs_raa { () => {
 353                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 354                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 355                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 356                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 357                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 358                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 359                 assert!(as_commitment_update.update_fee.is_none());
 360                 check_added_monitors!(nodes[0], 1);
 361         } }
 362
 363         macro_rules! handle_initial_raa { () => {
 364                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 365                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 366                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 367                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 368                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 369                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 370                 assert!(bs_second_commitment_update.update_fee.is_none());
 371                 check_added_monitors!(nodes[1], 1);
 372         } }
 373
 374         if (disconnect_count & 8) == 0 {
 375                 handle_bs_raa!();
 376
 377                 if disconnect_count & !disconnect_flags > 3 {
 378                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 379
 380                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 381                         assert!(bs_resp.1.is_none());
 382
 383                         assert!(as_resp.2.unwrap() == as_commitment_update);
 384                         assert!(bs_resp.2.is_none());
 385
 386                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 387                 }
 388
 389                 handle_initial_raa!();
 390
 391                 if disconnect_count & !disconnect_flags > 4 {
 392                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 393
 394                         assert!(as_resp.1.is_none());
 395                         assert!(bs_resp.1.is_none());
 396
 397                         assert!(as_resp.2.unwrap() == as_commitment_update);
 398                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 399                 }
 400         } else {
 401                 handle_initial_raa!();
 402
 403                 if disconnect_count & !disconnect_flags > 3 {
 404                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 405
 406                         assert!(as_resp.1.is_none());
 407                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 408
 409                         assert!(as_resp.2.is_none());
 410                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 411
 412                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 413                 }
 414
 415                 handle_bs_raa!();
 416
 417                 if disconnect_count & !disconnect_flags > 4 {
 418                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 419
 420                         assert!(as_resp.1.is_none());
 421                         assert!(bs_resp.1.is_none());
 422
 423                         assert!(as_resp.2.unwrap() == as_commitment_update);
 424                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 425                 }
 426         }
 427
 428         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 429         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 430         // No commitment_signed so get_event_msg's assert(len == 1) passes
 431         check_added_monitors!(nodes[0], 1);
 432
 433         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 434         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 435         // No commitment_signed so get_event_msg's assert(len == 1) passes
 436         check_added_monitors!(nodes[1], 1);
 437
 438         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 439         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 440         check_added_monitors!(nodes[1], 1);
 441
 442         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 443         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 444         check_added_monitors!(nodes[0], 1);
 445
 446         expect_pending_htlcs_forwardable!(nodes[1]);
 447
 448         let events_5 = nodes[1].node.get_and_clear_pending_events();
 449         assert_eq!(events_5.len(), 1);
 450         match events_5[0] {
 451                 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
 452                         assert_eq!(payment_hash_2, *payment_hash);
 453                         assert_eq!(*payment_secret, None);
 454                         assert_eq!(amt, 1000000);
 455                 },
 456                 _ => panic!("Unexpected event"),
 457         }
 458
 459         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 460 }
 461
 462 #[test]
 463 fn test_monitor_temporary_update_fail_a() {
 464         do_test_monitor_temporary_update_fail(0);
 465         do_test_monitor_temporary_update_fail(1);
 466         do_test_monitor_temporary_update_fail(2);
 467         do_test_monitor_temporary_update_fail(3);
 468         do_test_monitor_temporary_update_fail(4);
 469         do_test_monitor_temporary_update_fail(5);
 470 }
 471
 472 #[test]
 473 fn test_monitor_temporary_update_fail_b() {
 474         do_test_monitor_temporary_update_fail(2 | 8);
 475         do_test_monitor_temporary_update_fail(3 | 8);
 476         do_test_monitor_temporary_update_fail(4 | 8);
 477         do_test_monitor_temporary_update_fail(5 | 8);
 478 }
 479
 480 #[test]
 481 fn test_monitor_temporary_update_fail_c() {
 482         do_test_monitor_temporary_update_fail(1 | 16);
 483         do_test_monitor_temporary_update_fail(2 | 16);
 484         do_test_monitor_temporary_update_fail(3 | 16);
 485         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 486         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 487 }
 488
 489 #[test]
 490 fn test_monitor_update_fail_cs() {
 491         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 492         let chanmon_cfgs = create_chanmon_cfgs(2);
 493         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 494         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 495         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 496         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 497
 498         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 499         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 500         nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
 501         check_added_monitors!(nodes[0], 1);
 502
 503         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 504         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 505
 506         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 507         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 508         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 509         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 510         check_added_monitors!(nodes[1], 1);
 511         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 512
 513         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 514         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 515         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 516         check_added_monitors!(nodes[1], 0);
 517         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 518         assert_eq!(responses.len(), 2);
 519
 520         match responses[0] {
 521                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 522                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 523                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 524                         check_added_monitors!(nodes[0], 1);
 525                 },
 526                 _ => panic!("Unexpected event"),
 527         }
 528         match responses[1] {
 529                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 530                         assert!(updates.update_add_htlcs.is_empty());
 531                         assert!(updates.update_fulfill_htlcs.is_empty());
 532                         assert!(updates.update_fail_htlcs.is_empty());
 533                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 534                         assert!(updates.update_fee.is_none());
 535                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 536
 537                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 538                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 539                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 540                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 541                         check_added_monitors!(nodes[0], 1);
 542                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 543                 },
 544                 _ => panic!("Unexpected event"),
 545         }
 546
 547         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 548         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 549         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 550         check_added_monitors!(nodes[0], 0);
 551
 552         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 553         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 554         check_added_monitors!(nodes[1], 1);
 555
 556         expect_pending_htlcs_forwardable!(nodes[1]);
 557
 558         let events = nodes[1].node.get_and_clear_pending_events();
 559         assert_eq!(events.len(), 1);
 560         match events[0] {
 561                 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
 562                         assert_eq!(payment_hash, our_payment_hash);
 563                         assert_eq!(payment_secret, None);
 564                         assert_eq!(amt, 1000000);
 565                 },
 566                 _ => panic!("Unexpected event"),
 567         };
 568
 569         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 570 }
 571
 572 #[test]
 573 fn test_monitor_update_fail_no_rebroadcast() {
 574         // Tests handling of a monitor update failure when no message rebroadcasting on
 575         // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
 576         // fuzz tests.
 577         let chanmon_cfgs = create_chanmon_cfgs(2);
 578         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 579         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 580         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 581         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 582
 583         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 584         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 585         nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
 586         check_added_monitors!(nodes[0], 1);
 587
 588         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 589         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 590         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 591
 592         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 593         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 594         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 595         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 596         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 597         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 598         check_added_monitors!(nodes[1], 1);
 599
 600         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 601         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 602         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 603         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 604         check_added_monitors!(nodes[1], 0);
 605         expect_pending_htlcs_forwardable!(nodes[1]);
 606
 607         let events = nodes[1].node.get_and_clear_pending_events();
 608         assert_eq!(events.len(), 1);
 609         match events[0] {
 610                 Event::PaymentReceived { payment_hash, .. } => {
 611                         assert_eq!(payment_hash, our_payment_hash);
 612                 },
 613                 _ => panic!("Unexpected event"),
 614         }
 615
 616         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 617 }
 618
 619 #[test]
 620 fn test_monitor_update_raa_while_paused() {
 621         // Tests handling of an RAA while monitor updating has already been marked failed.
 622         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 623         let chanmon_cfgs = create_chanmon_cfgs(2);
 624         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 625         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 626         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 627         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
 628
 629         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 630
 631         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 632         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 633         nodes[0].node.send_payment(&route, our_payment_hash_1, &None).unwrap();
 634         check_added_monitors!(nodes[0], 1);
 635         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 636
 637         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 638         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 639         nodes[1].node.send_payment(&route, our_payment_hash_2, &None).unwrap();
 640         check_added_monitors!(nodes[1], 1);
 641         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 642
 643         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 644         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 645         check_added_monitors!(nodes[1], 1);
 646         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 647
 648         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 649         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 650         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 651         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 652         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 653         check_added_monitors!(nodes[0], 1);
 654
 655         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 656         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 657         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 658         check_added_monitors!(nodes[0], 1);
 659
 660         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 661         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 662         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 663         check_added_monitors!(nodes[0], 0);
 664
 665         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 666         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 667         check_added_monitors!(nodes[1], 1);
 668         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 669
 670         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 671         check_added_monitors!(nodes[1], 1);
 672         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 673
 674         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 675         check_added_monitors!(nodes[0], 1);
 676         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 677
 678         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 679         check_added_monitors!(nodes[0], 1);
 680         expect_pending_htlcs_forwardable!(nodes[0]);
 681         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 682
 683         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 684         check_added_monitors!(nodes[1], 1);
 685         expect_pending_htlcs_forwardable!(nodes[1]);
 686         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 687
 688         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 689         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 690 }
 691
 692 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 693         // Tests handling of a monitor update failure when processing an incoming RAA
 694         let chanmon_cfgs = create_chanmon_cfgs(3);
 695         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 696         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 697         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 698         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
 699         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
 700
 701         // Rebalance a bit so that we can send backwards from 2 to 1.
 702         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 703
 704         // Route a first payment that we'll fail backwards
 705         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 706
 707         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 708         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
 709         expect_pending_htlcs_forwardable!(nodes[2]);
 710         check_added_monitors!(nodes[2], 1);
 711
 712         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 713         assert!(updates.update_add_htlcs.is_empty());
 714         assert!(updates.update_fulfill_htlcs.is_empty());
 715         assert_eq!(updates.update_fail_htlcs.len(), 1);
 716         assert!(updates.update_fail_malformed_htlcs.is_empty());
 717         assert!(updates.update_fee.is_none());
 718         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 719
 720         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 721         check_added_monitors!(nodes[0], 0);
 722
 723         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 724         // holding cell.
 725         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 726         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 727         nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
 728         check_added_monitors!(nodes[0], 1);
 729
 730         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 731         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 732         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 733
 734         expect_pending_htlcs_forwardable!(nodes[1]);
 735         check_added_monitors!(nodes[1], 0);
 736         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 737
 738         // Now fail monitor updating.
 739         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 740         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 741         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 742         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 743         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 744         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 745         check_added_monitors!(nodes[1], 1);
 746
 747         // Attempt to forward a third payment but fail due to the second channel being unavailable
 748         // for forwarding.
 749
 750         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 751         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 752         nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
 753         check_added_monitors!(nodes[0], 1);
 754
 755         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 756         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 757         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 758         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 759         check_added_monitors!(nodes[1], 0);
 760
 761         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 762         assert_eq!(events_2.len(), 1);
 763         match events_2.remove(0) {
 764                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 765                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 766                         assert!(updates.update_fulfill_htlcs.is_empty());
 767                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 768                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 769                         assert!(updates.update_add_htlcs.is_empty());
 770                         assert!(updates.update_fee.is_none());
 771
 772                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 773                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 774
 775                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 776                         assert_eq!(msg_events.len(), 1);
 777                         match msg_events[0] {
 778                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 779                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 780                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 781                                 },
 782                                 _ => panic!("Unexpected event"),
 783                         }
 784
 785                         let events = nodes[0].node.get_and_clear_pending_events();
 786                         assert_eq!(events.len(), 1);
 787                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 788                                 assert_eq!(payment_hash, payment_hash_3);
 789                                 assert!(!rejected_by_dest);
 790                         } else { panic!("Unexpected event!"); }
 791                 },
 792                 _ => panic!("Unexpected event type!"),
 793         };
 794
 795         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 796                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 797                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 798                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 799                 nodes[2].node.send_payment(&route, payment_hash_4, &None).unwrap();
 800                 check_added_monitors!(nodes[2], 1);
 801
 802                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 803                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 804                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 805                 check_added_monitors!(nodes[1], 1);
 806                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 807                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 808                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 809                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 810                 (Some(payment_preimage_4), Some(payment_hash_4))
 811         } else { (None, None) };
 812
 813         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 814         // update_add update.
 815         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 816         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
 817         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 818         check_added_monitors!(nodes[1], 0);
 819         expect_pending_htlcs_forwardable!(nodes[1]);
 820         check_added_monitors!(nodes[1], 1);
 821
 822         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 823         if test_ignore_second_cs {
 824                 assert_eq!(events_3.len(), 3);
 825         } else {
 826                 assert_eq!(events_3.len(), 2);
 827         }
 828
 829         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 830         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 831         let messages_a = match events_3.pop().unwrap() {
 832                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 833                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 834                         assert!(updates.update_fulfill_htlcs.is_empty());
 835                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 836                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 837                         assert!(updates.update_add_htlcs.is_empty());
 838                         assert!(updates.update_fee.is_none());
 839                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 840                 },
 841                 _ => panic!("Unexpected event type!"),
 842         };
 843         let raa = if test_ignore_second_cs {
 844                 match events_3.remove(1) {
 845                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 846                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 847                                 Some(msg.clone())
 848                         },
 849                         _ => panic!("Unexpected event"),
 850                 }
 851         } else { None };
 852         let send_event_b = SendEvent::from_event(events_3.remove(0));
 853         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 854
 855         // Now deliver the new messages...
 856
 857         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 858         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 859         let events_4 = nodes[0].node.get_and_clear_pending_events();
 860         assert_eq!(events_4.len(), 1);
 861         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 862                 assert_eq!(payment_hash, payment_hash_1);
 863                 assert!(rejected_by_dest);
 864         } else { panic!("Unexpected event!"); }
 865
 866         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 867         if test_ignore_second_cs {
 868                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 869                 check_added_monitors!(nodes[2], 1);
 870                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 871                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 872                 check_added_monitors!(nodes[2], 1);
 873                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 874                 assert!(bs_cs.update_add_htlcs.is_empty());
 875                 assert!(bs_cs.update_fail_htlcs.is_empty());
 876                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 877                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 878                 assert!(bs_cs.update_fee.is_none());
 879
 880                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 881                 check_added_monitors!(nodes[1], 1);
 882                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 883                 assert!(as_cs.update_add_htlcs.is_empty());
 884                 assert!(as_cs.update_fail_htlcs.is_empty());
 885                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 886                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 887                 assert!(as_cs.update_fee.is_none());
 888
 889                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 890                 check_added_monitors!(nodes[1], 1);
 891                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 892
 893                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 894                 check_added_monitors!(nodes[2], 1);
 895                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 896
 897                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 898                 check_added_monitors!(nodes[2], 1);
 899                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 900
 901                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 902                 check_added_monitors!(nodes[1], 1);
 903                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 904         } else {
 905                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 906         }
 907
 908         expect_pending_htlcs_forwardable!(nodes[2]);
 909
 910         let events_6 = nodes[2].node.get_and_clear_pending_events();
 911         assert_eq!(events_6.len(), 1);
 912         match events_6[0] {
 913                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 914                 _ => panic!("Unexpected event"),
 915         };
 916
 917         if test_ignore_second_cs {
 918                 expect_pending_htlcs_forwardable!(nodes[1]);
 919                 check_added_monitors!(nodes[1], 1);
 920
 921                 send_event = SendEvent::from_node(&nodes[1]);
 922                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 923                 assert_eq!(send_event.msgs.len(), 1);
 924                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 925                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 926
 927                 expect_pending_htlcs_forwardable!(nodes[0]);
 928
 929                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 930                 assert_eq!(events_9.len(), 1);
 931                 match events_9[0] {
 932                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 933                         _ => panic!("Unexpected event"),
 934                 };
 935                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 936         }
 937
 938         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 939 }
 940
 941 #[test]
 942 fn test_monitor_update_fail_raa() {
 943         do_test_monitor_update_fail_raa(false);
 944         do_test_monitor_update_fail_raa(true);
 945 }
 946
 947 #[test]
 948 fn test_monitor_update_fail_reestablish() {
 949         // Simple test for message retransmission after monitor update failure on
 950         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 951         // HTLCs).
 952         let chanmon_cfgs = create_chanmon_cfgs(3);
 953         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 954         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 955         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 956         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
 957         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
 958
 959         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 960
 961         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 962         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 963
 964         assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
 965         check_added_monitors!(nodes[2], 1);
 966         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 967         assert!(updates.update_add_htlcs.is_empty());
 968         assert!(updates.update_fail_htlcs.is_empty());
 969         assert!(updates.update_fail_malformed_htlcs.is_empty());
 970         assert!(updates.update_fee.is_none());
 971         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 972         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 973         check_added_monitors!(nodes[1], 1);
 974         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 975         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 976
 977         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 978         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 979         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 980
 981         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 982         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 983
 984         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 985
 986         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 987         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 988         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 989         check_added_monitors!(nodes[1], 1);
 990
 991         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 992         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 993
 994         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 995         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 996
 997         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 998         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 999
1000         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1001
1002         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1003         check_added_monitors!(nodes[1], 0);
1004         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1005
1006         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1007         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1008         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1009         check_added_monitors!(nodes[1], 0);
1010
1011         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1012         assert!(updates.update_add_htlcs.is_empty());
1013         assert!(updates.update_fail_htlcs.is_empty());
1014         assert!(updates.update_fail_malformed_htlcs.is_empty());
1015         assert!(updates.update_fee.is_none());
1016         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1017         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1018         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1019
1020         let events = nodes[0].node.get_and_clear_pending_events();
1021         assert_eq!(events.len(), 1);
1022         match events[0] {
1023                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1024                 _ => panic!("Unexpected event"),
1025         }
1026 }
1027
1028 #[test]
1029 fn raa_no_response_awaiting_raa_state() {
1030         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1031         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1032         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1033         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1034         let chanmon_cfgs = create_chanmon_cfgs(2);
1035         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1036         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1037         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1038         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1039
1040         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1041         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1042         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1043         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1044
1045         // Queue up two payments - one will be delivered right away, one immediately goes into the
1046         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1047         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1048         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1049         // generation during RAA while in monitor-update-failed state.
1050         nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1051         check_added_monitors!(nodes[0], 1);
1052         nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1053         check_added_monitors!(nodes[0], 0);
1054
1055         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1056         assert_eq!(events.len(), 1);
1057         let payment_event = SendEvent::from_event(events.pop().unwrap());
1058         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1059         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1060         check_added_monitors!(nodes[1], 1);
1061
1062         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1063         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1064         check_added_monitors!(nodes[0], 1);
1065         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1066         assert_eq!(events.len(), 1);
1067         let payment_event = SendEvent::from_event(events.pop().unwrap());
1068
1069         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1070         check_added_monitors!(nodes[0], 1);
1071         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1072
1073         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1074         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1075         // then restore channel monitor updates.
1076         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1077         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1078         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1079         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1080         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1081         check_added_monitors!(nodes[1], 1);
1082
1083         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1084         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1085         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1086         check_added_monitors!(nodes[1], 1);
1087
1088         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1089         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1090         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1091         // nodes[1] should be AwaitingRAA here!
1092         check_added_monitors!(nodes[1], 0);
1093         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1094         expect_pending_htlcs_forwardable!(nodes[1]);
1095         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1096
1097         // We send a third payment here, which is somewhat of a redundant test, but the
1098         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1099         // commitment transaction states) whereas here we can explicitly check for it.
1100         nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
1101         check_added_monitors!(nodes[0], 0);
1102         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1103
1104         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1105         check_added_monitors!(nodes[0], 1);
1106         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1107         assert_eq!(events.len(), 1);
1108         let payment_event = SendEvent::from_event(events.pop().unwrap());
1109
1110         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1111         check_added_monitors!(nodes[0], 1);
1112         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1113
1114         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1115         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1116         check_added_monitors!(nodes[1], 1);
1117         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1118
1119         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1120         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1121         check_added_monitors!(nodes[1], 1);
1122         expect_pending_htlcs_forwardable!(nodes[1]);
1123         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1124         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1125
1126         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1127         check_added_monitors!(nodes[0], 1);
1128
1129         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1130         check_added_monitors!(nodes[0], 1);
1131         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1132
1133         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1134         check_added_monitors!(nodes[1], 1);
1135         expect_pending_htlcs_forwardable!(nodes[1]);
1136         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1137
1138         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1139         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1140         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1141 }
1142
1143 #[test]
1144 fn claim_while_disconnected_monitor_update_fail() {
1145         // Test for claiming a payment while disconnected and then having the resulting
1146         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1147         // contrived case for nodes with network instability.
1148         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1149         // code introduced a regression in this test (specifically, this caught a removal of the
1150         // channel_reestablish handling ensuring the order was sensical given the messages used).
1151         let chanmon_cfgs = create_chanmon_cfgs(2);
1152         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1153         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1154         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1155         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1156
1157         // Forward a payment for B to claim
1158         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1159
1160         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1161         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1162
1163         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1164         check_added_monitors!(nodes[1], 1);
1165
1166         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1167         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1168
1169         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1170         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1171
1172         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1173         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1174
1175         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1176         // update.
1177         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1178
1179         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1180         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1181         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1182         check_added_monitors!(nodes[1], 1);
1183         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1184
1185         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1186         // the monitor still failed
1187         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1188         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1189         nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1190         check_added_monitors!(nodes[0], 1);
1191
1192         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1193         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1194         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1195         check_added_monitors!(nodes[1], 1);
1196         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1197         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1198         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1199         // until we've channel_monitor_update'd and updated for the new commitment transaction.
1200
1201         // Now un-fail the monitor, which will result in B sending its original commitment update,
1202         // receiving the commitment update from A, and the resulting commitment dances.
1203         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1204         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1205         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1206         check_added_monitors!(nodes[1], 0);
1207
1208         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1209         assert_eq!(bs_msgs.len(), 2);
1210
1211         match bs_msgs[0] {
1212                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1213                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1214                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1215                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1216                         check_added_monitors!(nodes[0], 1);
1217
1218                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1219                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1220                         check_added_monitors!(nodes[1], 1);
1221                 },
1222                 _ => panic!("Unexpected event"),
1223         }
1224
1225         match bs_msgs[1] {
1226                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1227                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1228                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1229                         check_added_monitors!(nodes[0], 1);
1230                 },
1231                 _ => panic!("Unexpected event"),
1232         }
1233
1234         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1235
1236         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1237         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1238         check_added_monitors!(nodes[0], 1);
1239         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1240
1241         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1242         check_added_monitors!(nodes[1], 1);
1243         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1244         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1245         check_added_monitors!(nodes[1], 1);
1246
1247         expect_pending_htlcs_forwardable!(nodes[1]);
1248         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1249
1250         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1251         check_added_monitors!(nodes[0], 1);
1252
1253         let events = nodes[0].node.get_and_clear_pending_events();
1254         assert_eq!(events.len(), 1);
1255         match events[0] {
1256                 Event::PaymentSent { ref payment_preimage } => {
1257                         assert_eq!(*payment_preimage, payment_preimage_1);
1258                 },
1259                 _ => panic!("Unexpected event"),
1260         }
1261
1262         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1263 }
1264
1265 #[test]
1266 fn monitor_failed_no_reestablish_response() {
1267         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1268         // response to a commitment_signed.
1269         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1270         // debug_assert!() failure in channel_reestablish handling.
1271         let chanmon_cfgs = create_chanmon_cfgs(2);
1272         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1273         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1274         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1275         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1276
1277         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1278         // on receipt).
1279         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1280         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1281         nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1282         check_added_monitors!(nodes[0], 1);
1283
1284         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1285         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1286         assert_eq!(events.len(), 1);
1287         let payment_event = SendEvent::from_event(events.pop().unwrap());
1288         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1289         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1290         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1291         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1292         check_added_monitors!(nodes[1], 1);
1293
1294         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1295         // is still failing to update monitors.
1296         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1297         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1298
1299         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1300         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1301
1302         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1303         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1304
1305         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1306         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1307
1308         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1309         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1310         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1311         check_added_monitors!(nodes[1], 0);
1312         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1313
1314         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1315         check_added_monitors!(nodes[0], 1);
1316         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1317         check_added_monitors!(nodes[0], 1);
1318
1319         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1320         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1321         check_added_monitors!(nodes[1], 1);
1322
1323         expect_pending_htlcs_forwardable!(nodes[1]);
1324         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1325
1326         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1327 }
1328
1329 #[test]
1330 fn first_message_on_recv_ordering() {
1331         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1332         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1333         // a commitment_signed which needs to send an RAA first.
1334         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1335         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1336         // response. To do this, we start routing two payments, with the final RAA for the first being
1337         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1338         // have no pending response but will want to send a RAA/CS (with the updates for the second
1339         // payment applied).
1340         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1341         let chanmon_cfgs = create_chanmon_cfgs(2);
1342         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1343         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1344         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1345         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1346
1347         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1348         // can deliver it and fail the monitor update.
1349         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1350         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1351         nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1352         check_added_monitors!(nodes[0], 1);
1353
1354         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1355         assert_eq!(events.len(), 1);
1356         let payment_event = SendEvent::from_event(events.pop().unwrap());
1357         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1358         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1359         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1360         check_added_monitors!(nodes[1], 1);
1361         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1362
1363         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1364         check_added_monitors!(nodes[0], 1);
1365         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1366         check_added_monitors!(nodes[0], 1);
1367
1368         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1369
1370         // Route the second payment, generating an update_add_htlc/commitment_signed
1371         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1372         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1373         nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1374         check_added_monitors!(nodes[0], 1);
1375         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1376         assert_eq!(events.len(), 1);
1377         let payment_event = SendEvent::from_event(events.pop().unwrap());
1378         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1379
1380         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1381
1382         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1383         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1384         // to the next message also tests resetting the delivery order.
1385         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1386         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1387         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1388         check_added_monitors!(nodes[1], 1);
1389
1390         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1391         // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1392         // appropriate HTLC acceptance).
1393         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1394         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1395         check_added_monitors!(nodes[1], 1);
1396         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1397         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1398
1399         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1400         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1401         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1402         check_added_monitors!(nodes[1], 0);
1403
1404         expect_pending_htlcs_forwardable!(nodes[1]);
1405         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1406
1407         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1408         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1409         check_added_monitors!(nodes[0], 1);
1410         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1411         check_added_monitors!(nodes[0], 1);
1412
1413         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1414         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1415         check_added_monitors!(nodes[1], 1);
1416
1417         expect_pending_htlcs_forwardable!(nodes[1]);
1418         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1419
1420         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1421         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1422 }
1423
1424 #[test]
1425 fn test_monitor_update_fail_claim() {
1426         // Basic test for monitor update failures when processing claim_funds calls.
1427         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1428         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1429         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1430         // updating and claim the payment on B.
1431         let chanmon_cfgs = create_chanmon_cfgs(3);
1432         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1433         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1434         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1435         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1436         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1437
1438         // Rebalance a bit so that we can send backwards from 3 to 2.
1439         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1440
1441         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1442
1443         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1444         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1445         check_added_monitors!(nodes[1], 1);
1446
1447         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1448         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1449         nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1450         check_added_monitors!(nodes[2], 1);
1451
1452         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1453         // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1454         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1455
1456         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1457         assert_eq!(events.len(), 1);
1458         let payment_event = SendEvent::from_event(events.pop().unwrap());
1459         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1460         let events = nodes[1].node.get_and_clear_pending_msg_events();
1461         assert_eq!(events.len(), 0);
1462         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1463         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1464
1465         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1466         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1467         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1468
1469         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1470         assert_eq!(msg_events.len(), 1);
1471         match msg_events[0] {
1472                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1473                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1474                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1475                 },
1476                 _ => panic!("Unexpected event"),
1477         }
1478
1479         let events = nodes[2].node.get_and_clear_pending_events();
1480         assert_eq!(events.len(), 1);
1481         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1482                 assert_eq!(payment_hash, payment_hash_2);
1483                 assert!(!rejected_by_dest);
1484         } else { panic!("Unexpected event!"); }
1485
1486         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1487         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1488         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1489         check_added_monitors!(nodes[1], 0);
1490
1491         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1492         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1493         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1494
1495         let events = nodes[0].node.get_and_clear_pending_events();
1496         assert_eq!(events.len(), 1);
1497         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1498                 assert_eq!(payment_preimage, payment_preimage_1);
1499         } else { panic!("Unexpected event!"); }
1500 }
1501
1502 #[test]
1503 fn test_monitor_update_on_pending_forwards() {
1504         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1505         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1506         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1507         // from C to A will be pending a forward to A.
1508         let chanmon_cfgs = create_chanmon_cfgs(3);
1509         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1510         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1511         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1512         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1513         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1514
1515         // Rebalance a bit so that we can send backwards from 3 to 1.
1516         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1517
1518         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1519         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1520         expect_pending_htlcs_forwardable!(nodes[2]);
1521         check_added_monitors!(nodes[2], 1);
1522
1523         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1524         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1525         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1526         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1527
1528         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1529         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1530         nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1531         check_added_monitors!(nodes[2], 1);
1532
1533         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1534         assert_eq!(events.len(), 1);
1535         let payment_event = SendEvent::from_event(events.pop().unwrap());
1536         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1537         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1538
1539         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1540         expect_pending_htlcs_forwardable!(nodes[1]);
1541         check_added_monitors!(nodes[1], 1);
1542         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1543         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1544
1545         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1546         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1547         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1548         check_added_monitors!(nodes[1], 0);
1549
1550         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1551         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1552         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1553         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1554
1555         let events = nodes[0].node.get_and_clear_pending_events();
1556         assert_eq!(events.len(), 2);
1557         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1558                 assert_eq!(payment_hash, payment_hash_1);
1559                 assert!(rejected_by_dest);
1560         } else { panic!("Unexpected event!"); }
1561         match events[1] {
1562                 Event::PendingHTLCsForwardable { .. } => { },
1563                 _ => panic!("Unexpected event"),
1564         };
1565         nodes[0].node.process_pending_htlc_forwards();
1566         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1567
1568         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1569 }
1570
1571 #[test]
1572 fn monitor_update_claim_fail_no_response() {
1573         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1574         // to channel being AwaitingRAA).
1575         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1576         // code was broken.
1577         let chanmon_cfgs = create_chanmon_cfgs(2);
1578         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1579         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1580         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1581         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1582
1583         // Forward a payment for B to claim
1584         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1585
1586         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1587         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1588         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1589         nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1590         check_added_monitors!(nodes[0], 1);
1591
1592         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1593         assert_eq!(events.len(), 1);
1594         let payment_event = SendEvent::from_event(events.pop().unwrap());
1595         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1596         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1597
1598         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1599         assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1600         check_added_monitors!(nodes[1], 1);
1601         let events = nodes[1].node.get_and_clear_pending_msg_events();
1602         assert_eq!(events.len(), 0);
1603         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1604
1605         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1606         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1607         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1608         check_added_monitors!(nodes[1], 0);
1609         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1610
1611         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1612         check_added_monitors!(nodes[1], 1);
1613         expect_pending_htlcs_forwardable!(nodes[1]);
1614         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1615
1616         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1617         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1618         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1619
1620         let events = nodes[0].node.get_and_clear_pending_events();
1621         assert_eq!(events.len(), 1);
1622         match events[0] {
1623                 Event::PaymentSent { ref payment_preimage } => {
1624                         assert_eq!(*payment_preimage, payment_preimage_1);
1625                 },
1626                 _ => panic!("Unexpected event"),
1627         }
1628
1629         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1630 }
1631
1632 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1633 // restore_b_before_conf has no meaning if !confirm_a_first
1634 fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf: bool) {
1635         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1636         // the channel setup happily after the update is restored.
1637         let chanmon_cfgs = create_chanmon_cfgs(2);
1638         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1639         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1640         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1641
1642         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1643         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1644         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1645
1646         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1647
1648         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1649         check_added_monitors!(nodes[0], 0);
1650
1651         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1652         let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1653         let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1654         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1655         check_added_monitors!(nodes[1], 1);
1656
1657         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1658         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1659         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1660         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1661         check_added_monitors!(nodes[0], 1);
1662         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1663         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1664         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1665         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1666         check_added_monitors!(nodes[0], 0);
1667
1668         let events = nodes[0].node.get_and_clear_pending_events();
1669         assert_eq!(events.len(), 1);
1670         match events[0] {
1671                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1672                         assert_eq!(user_channel_id, 43);
1673                         assert_eq!(*funding_txo, funding_output);
1674                 },
1675                 _ => panic!("Unexpected event"),
1676         };
1677
1678         if confirm_a_first {
1679                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1680                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1681         } else {
1682                 assert!(!restore_b_before_conf);
1683                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1684                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1685         }
1686
1687         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1688         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1689         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1690         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1691         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1692         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1693
1694         if !restore_b_before_conf {
1695                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1696                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1697                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1698         }
1699
1700         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1701         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1702         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1703         check_added_monitors!(nodes[1], 0);
1704
1705         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1706                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1707
1708                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1709                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1710                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1711         } else {
1712                 if restore_b_before_conf {
1713                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1714                 }
1715                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1716                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1717         };
1718         for node in nodes.iter() {
1719                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1720                 node.router.handle_channel_update(&as_update).unwrap();
1721                 node.router.handle_channel_update(&bs_update).unwrap();
1722         }
1723
1724         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1725         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1726 }
1727
1728 #[test]
1729 fn during_funding_monitor_fail() {
1730         do_during_funding_monitor_fail(true, true);
1731         do_during_funding_monitor_fail(true, false);
1732         do_during_funding_monitor_fail(false, false);
1733 }
1734
1735 #[test]
1736 fn test_path_paused_mpp() {
1737         // Simple test of sending a multi-part payment where one path is currently blocked awaiting
1738         // monitor update
1739         let chanmon_cfgs = create_chanmon_cfgs(4);
1740         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1741         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
1742         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1743
1744         let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1745         let (chan_2_ann, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known());
1746         let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1747         let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1748
1749         let (payment_preimage, payment_hash) = get_payment_preimage_hash!(&nodes[0]);
1750         let payment_secret = PaymentSecret([0xdb; 32]);
1751         let mut route = nodes[0].router.get_route(&nodes[3].node.get_our_node_id(), None, &[], 100000, TEST_FINAL_CLTV).unwrap();
1752
1753         // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
1754         let path = route.paths[0].clone();
1755         route.paths.push(path);
1756         route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
1757         route.paths[0][0].short_channel_id = chan_1_id;
1758         route.paths[0][1].short_channel_id = chan_3_id;
1759         route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
1760         route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
1761         route.paths[1][1].short_channel_id = chan_4_id;
1762
1763         // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
1764         // (for the path 0 -> 2 -> 3) fails.
1765         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1766         *nodes[0].chan_monitor.next_update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1767
1768         // Now check that we get the right return value, indicating that the first path succeeded but
1769         // the second got a MonitorUpdateFailed err. This implies PaymentSendFailure::PartialFailure as
1770         // some paths succeeded, preventing retry.
1771         if let Err(PaymentSendFailure::PartialFailure(results)) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
1772                 assert_eq!(results.len(), 2);
1773                 if let Ok(()) = results[0] {} else { panic!(); }
1774                 if let Err(APIError::MonitorUpdateFailed) = results[1] {} else { panic!(); }
1775         } else { panic!(); }
1776         check_added_monitors!(nodes[0], 2);
1777         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1778
1779         // Pass the first HTLC of the payment along to nodes[3].
1780         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1781         assert_eq!(events.len(), 1);
1782         pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 0, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), false);
1783
1784         // And check that, after we successfully update the monitor for chan_2 we can pass the second
1785         // HTLC along to nodes[3] and claim the whole payment back to nodes[0].
1786         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2_id).unwrap().clone();
1787         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1788         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1789         assert_eq!(events.len(), 1);
1790         pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), true);
1791
1792         claim_payment_along_route_with_secret(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage, Some(payment_secret), 200_000);
1793 }