_ Git - rust-lightning/blob - lightning/src/ln/chanmon_update_fail_tests.rs

   1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
   2 //! monitor updates.
   3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
   4 //! here. See also the chanmon_fail_consistency fuzz test.
   5
   6 use chain::transaction::OutPoint;
   7 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
   8 use ln::channelmonitor::ChannelMonitorUpdateErr;
   9 use ln::features::InitFeatures;
  10 use ln::msgs;
  11 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
  12 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
  13 use util::errors::APIError;
  14
  15 use bitcoin_hashes::sha256::Hash as Sha256;
  16 use bitcoin_hashes::Hash;
  17
  18 use ln::functional_test_utils::*;
  19
  20 #[test]
  21 fn test_simple_monitor_permanent_update_fail() {
  22         // Test that we handle a simple permanent monitor update failure
  23         let chanmon_cfgs = create_chanmon_cfgs(2);
  24         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  25         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  26         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  27         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
  28
  29         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  30         let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  31
  32         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
  33         if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
  34         check_added_monitors!(nodes[0], 1);
  35
  36         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  37         assert_eq!(events_1.len(), 2);
  38         match events_1[0] {
  39                 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
  40                 _ => panic!("Unexpected event"),
  41         };
  42         match events_1[1] {
  43                 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
  44                 _ => panic!("Unexpected event"),
  45         };
  46
  47         // TODO: Once we hit the chain with the failure transaction we should check that we get a
  48         // PaymentFailed event
  49
  50         assert_eq!(nodes[0].node.list_channels().len(), 0);
  51 }
  52
  53 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
  54         // Test that we can recover from a simple temporary monitor update failure optionally with
  55         // a disconnect in between
  56         let chanmon_cfgs = create_chanmon_cfgs(2);
  57         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  58         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  59         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  60         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
  61
  62         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
  63         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
  64
  65         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
  66         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
  67         check_added_monitors!(nodes[0], 1);
  68
  69         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
  70         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
  71         assert_eq!(nodes[0].node.list_channels().len(), 1);
  72
  73         if disconnect {
  74                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  75                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  76                 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  77         }
  78
  79         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
  80         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
  81         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
  82         check_added_monitors!(nodes[0], 0);
  83
  84         let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
  85         assert_eq!(events_2.len(), 1);
  86         let payment_event = SendEvent::from_event(events_2.pop().unwrap());
  87         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
  88         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
  89         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
  90
  91         expect_pending_htlcs_forwardable!(nodes[1]);
  92
  93         let events_3 = nodes[1].node.get_and_clear_pending_events();
  94         assert_eq!(events_3.len(), 1);
  95         match events_3[0] {
  96                 Event::PaymentReceived { ref payment_hash, amt } => {
  97                         assert_eq!(payment_hash_1, *payment_hash);
  98                         assert_eq!(amt, 1000000);
  99                 },
 100                 _ => panic!("Unexpected event"),
 101         }
 102
 103         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 104
 105         // Now set it to failed again...
 106         let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
 107         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 108         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
 109         check_added_monitors!(nodes[0], 1);
 110
 111         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 112         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 113         assert_eq!(nodes[0].node.list_channels().len(), 1);
 114
 115         if disconnect {
 116                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 117                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 118                 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 119         }
 120
 121         // ...and make sure we can force-close a frozen channel
 122         nodes[0].node.force_close_channel(&channel_id);
 123         check_added_monitors!(nodes[0], 0);
 124         check_closed_broadcast!(nodes[0], false);
 125
 126         // TODO: Once we hit the chain with the failure transaction we should check that we get a
 127         // PaymentFailed event
 128
 129         assert_eq!(nodes[0].node.list_channels().len(), 0);
 130 }
 131
 132 #[test]
 133 fn test_simple_monitor_temporary_update_fail() {
 134         do_test_simple_monitor_temporary_update_fail(false);
 135         do_test_simple_monitor_temporary_update_fail(true);
 136 }
 137
 138 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
 139         let disconnect_flags = 8 | 16;
 140
 141         // Test that we can recover from a temporary monitor update failure with some in-flight
 142         // HTLCs going on at the same time potentially with some disconnection thrown in.
 143         // * First we route a payment, then get a temporary monitor update failure when trying to
 144         //   route a second payment. We then claim the first payment.
 145         // * If disconnect_count is set, we will disconnect at this point (which is likely as
 146         //   TemporaryFailure likely indicates net disconnect which resulted in failing to update
 147         //   the ChannelMonitor on a watchtower).
 148         // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
 149         //   immediately, otherwise we wait disconnect and deliver them via the reconnect
 150         //   channel_reestablish processing (ie disconnect_count & 16 makes no sense if
 151         //   disconnect_count & !disconnect_flags is 0).
 152         // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
 153         //   through message sending, potentially disconnect/reconnecting multiple times based on
 154         //   disconnect_count, to get the update_fulfill_htlc through.
 155         // * We then walk through more message exchanges to get the original update_add_htlc
 156         //   through, swapping message ordering based on disconnect_count & 8 and optionally
 157         //   disconnect/reconnecting based on disconnect_count.
 158         let chanmon_cfgs = create_chanmon_cfgs(2);
 159         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 160         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 161         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 162         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
 163
 164         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 165
 166         // Now try to send a second payment which will fail to send
 167         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 168         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 169
 170         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 171         if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
 172         check_added_monitors!(nodes[0], 1);
 173
 174         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 175         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 176         assert_eq!(nodes[0].node.list_channels().len(), 1);
 177
 178         // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
 179         // but nodes[0] won't respond since it is frozen.
 180         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
 181         check_added_monitors!(nodes[1], 1);
 182         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 183         assert_eq!(events_2.len(), 1);
 184         let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
 185                 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
 186                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 187                         assert!(update_add_htlcs.is_empty());
 188                         assert_eq!(update_fulfill_htlcs.len(), 1);
 189                         assert!(update_fail_htlcs.is_empty());
 190                         assert!(update_fail_malformed_htlcs.is_empty());
 191                         assert!(update_fee.is_none());
 192
 193                         if (disconnect_count & 16) == 0 {
 194                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 195                                 let events_3 = nodes[0].node.get_and_clear_pending_events();
 196                                 assert_eq!(events_3.len(), 1);
 197                                 match events_3[0] {
 198                                         Event::PaymentSent { ref payment_preimage } => {
 199                                                 assert_eq!(*payment_preimage, payment_preimage_1);
 200                                         },
 201                                         _ => panic!("Unexpected event"),
 202                                 }
 203
 204                                 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
 205                                 check_added_monitors!(nodes[0], 1);
 206                                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 207                                 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 208                         }
 209
 210                         (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
 211                 },
 212                 _ => panic!("Unexpected event"),
 213         };
 214
 215         if disconnect_count & !disconnect_flags > 0 {
 216                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 217                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 218         }
 219
 220         // Now fix monitor updating...
 221         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 222         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 223         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 224         check_added_monitors!(nodes[0], 0);
 225
 226         macro_rules! disconnect_reconnect_peers { () => { {
 227                 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 228                 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 229
 230                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 231                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 232                 assert_eq!(reestablish_1.len(), 1);
 233                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 234                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 235                 assert_eq!(reestablish_2.len(), 1);
 236
 237                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 238                 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 239                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 240                 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 241
 242                 assert!(as_resp.0.is_none());
 243                 assert!(bs_resp.0.is_none());
 244
 245                 (reestablish_1, reestablish_2, as_resp, bs_resp)
 246         } } }
 247
 248         let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
 249                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
 250                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 251
 252                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 253                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 254                 assert_eq!(reestablish_1.len(), 1);
 255                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 256                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 257                 assert_eq!(reestablish_2.len(), 1);
 258
 259                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 260                 check_added_monitors!(nodes[0], 0);
 261                 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
 262                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 263                 check_added_monitors!(nodes[1], 0);
 264                 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
 265
 266                 assert!(as_resp.0.is_none());
 267                 assert!(bs_resp.0.is_none());
 268
 269                 assert!(bs_resp.1.is_none());
 270                 if (disconnect_count & 16) == 0 {
 271                         assert!(bs_resp.2.is_none());
 272
 273                         assert!(as_resp.1.is_some());
 274                         assert!(as_resp.2.is_some());
 275                         assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
 276                 } else {
 277                         assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
 278                         assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
 279                         assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
 280                         assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
 281                         assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
 282                         assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
 283
 284                         assert!(as_resp.1.is_none());
 285
 286                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
 287                         let events_3 = nodes[0].node.get_and_clear_pending_events();
 288                         assert_eq!(events_3.len(), 1);
 289                         match events_3[0] {
 290                                 Event::PaymentSent { ref payment_preimage } => {
 291                                         assert_eq!(*payment_preimage, payment_preimage_1);
 292                                 },
 293                                 _ => panic!("Unexpected event"),
 294                         }
 295
 296                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
 297                         let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 298                         // No commitment_signed so get_event_msg's assert(len == 1) passes
 299                         check_added_monitors!(nodes[0], 1);
 300
 301                         as_resp.1 = Some(as_resp_raa);
 302                         bs_resp.2 = None;
 303                 }
 304
 305                 if disconnect_count & !disconnect_flags > 1 {
 306                         let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
 307
 308                         if (disconnect_count & 16) == 0 {
 309                                 assert!(reestablish_1 == second_reestablish_1);
 310                                 assert!(reestablish_2 == second_reestablish_2);
 311                         }
 312                         assert!(as_resp == second_as_resp);
 313                         assert!(bs_resp == second_bs_resp);
 314                 }
 315
 316                 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
 317         } else {
 318                 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
 319                 assert_eq!(events_4.len(), 2);
 320                 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
 321                         MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
 322                                 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 323                                 msg.clone()
 324                         },
 325                         _ => panic!("Unexpected event"),
 326                 })
 327         };
 328
 329         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
 330
 331         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 332         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
 333         let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 334         // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
 335         check_added_monitors!(nodes[1], 1);
 336
 337         if disconnect_count & !disconnect_flags > 2 {
 338                 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 339
 340                 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 341                 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 342
 343                 assert!(as_resp.2.is_none());
 344                 assert!(bs_resp.2.is_none());
 345         }
 346
 347         let as_commitment_update;
 348         let bs_second_commitment_update;
 349
 350         macro_rules! handle_bs_raa { () => {
 351                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
 352                 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 353                 assert!(as_commitment_update.update_add_htlcs.is_empty());
 354                 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
 355                 assert!(as_commitment_update.update_fail_htlcs.is_empty());
 356                 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
 357                 assert!(as_commitment_update.update_fee.is_none());
 358                 check_added_monitors!(nodes[0], 1);
 359         } }
 360
 361         macro_rules! handle_initial_raa { () => {
 362                 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
 363                 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 364                 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
 365                 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
 366                 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
 367                 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
 368                 assert!(bs_second_commitment_update.update_fee.is_none());
 369                 check_added_monitors!(nodes[1], 1);
 370         } }
 371
 372         if (disconnect_count & 8) == 0 {
 373                 handle_bs_raa!();
 374
 375                 if disconnect_count & !disconnect_flags > 3 {
 376                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 377
 378                         assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
 379                         assert!(bs_resp.1.is_none());
 380
 381                         assert!(as_resp.2.unwrap() == as_commitment_update);
 382                         assert!(bs_resp.2.is_none());
 383
 384                         assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 385                 }
 386
 387                 handle_initial_raa!();
 388
 389                 if disconnect_count & !disconnect_flags > 4 {
 390                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 391
 392                         assert!(as_resp.1.is_none());
 393                         assert!(bs_resp.1.is_none());
 394
 395                         assert!(as_resp.2.unwrap() == as_commitment_update);
 396                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 397                 }
 398         } else {
 399                 handle_initial_raa!();
 400
 401                 if disconnect_count & !disconnect_flags > 3 {
 402                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 403
 404                         assert!(as_resp.1.is_none());
 405                         assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
 406
 407                         assert!(as_resp.2.is_none());
 408                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 409
 410                         assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
 411                 }
 412
 413                 handle_bs_raa!();
 414
 415                 if disconnect_count & !disconnect_flags > 4 {
 416                         let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
 417
 418                         assert!(as_resp.1.is_none());
 419                         assert!(bs_resp.1.is_none());
 420
 421                         assert!(as_resp.2.unwrap() == as_commitment_update);
 422                         assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
 423                 }
 424         }
 425
 426         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
 427         let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 428         // No commitment_signed so get_event_msg's assert(len == 1) passes
 429         check_added_monitors!(nodes[0], 1);
 430
 431         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
 432         let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 433         // No commitment_signed so get_event_msg's assert(len == 1) passes
 434         check_added_monitors!(nodes[1], 1);
 435
 436         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
 437         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 438         check_added_monitors!(nodes[1], 1);
 439
 440         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
 441         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 442         check_added_monitors!(nodes[0], 1);
 443
 444         expect_pending_htlcs_forwardable!(nodes[1]);
 445
 446         let events_5 = nodes[1].node.get_and_clear_pending_events();
 447         assert_eq!(events_5.len(), 1);
 448         match events_5[0] {
 449                 Event::PaymentReceived { ref payment_hash, amt } => {
 450                         assert_eq!(payment_hash_2, *payment_hash);
 451                         assert_eq!(amt, 1000000);
 452                 },
 453                 _ => panic!("Unexpected event"),
 454         }
 455
 456         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
 457 }
 458
 459 #[test]
 460 fn test_monitor_temporary_update_fail_a() {
 461         do_test_monitor_temporary_update_fail(0);
 462         do_test_monitor_temporary_update_fail(1);
 463         do_test_monitor_temporary_update_fail(2);
 464         do_test_monitor_temporary_update_fail(3);
 465         do_test_monitor_temporary_update_fail(4);
 466         do_test_monitor_temporary_update_fail(5);
 467 }
 468
 469 #[test]
 470 fn test_monitor_temporary_update_fail_b() {
 471         do_test_monitor_temporary_update_fail(2 | 8);
 472         do_test_monitor_temporary_update_fail(3 | 8);
 473         do_test_monitor_temporary_update_fail(4 | 8);
 474         do_test_monitor_temporary_update_fail(5 | 8);
 475 }
 476
 477 #[test]
 478 fn test_monitor_temporary_update_fail_c() {
 479         do_test_monitor_temporary_update_fail(1 | 16);
 480         do_test_monitor_temporary_update_fail(2 | 16);
 481         do_test_monitor_temporary_update_fail(3 | 16);
 482         do_test_monitor_temporary_update_fail(2 | 8 | 16);
 483         do_test_monitor_temporary_update_fail(3 | 8 | 16);
 484 }
 485
 486 #[test]
 487 fn test_monitor_update_fail_cs() {
 488         // Tests handling of a monitor update failure when processing an incoming commitment_signed
 489         let chanmon_cfgs = create_chanmon_cfgs(2);
 490         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 491         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 492         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 493         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
 494
 495         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 496         let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 497         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 498         check_added_monitors!(nodes[0], 1);
 499
 500         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 501         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 502
 503         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 504         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
 505         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 506         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 507         check_added_monitors!(nodes[1], 1);
 508         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 509
 510         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 511         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 512         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 513         check_added_monitors!(nodes[1], 0);
 514         let responses = nodes[1].node.get_and_clear_pending_msg_events();
 515         assert_eq!(responses.len(), 2);
 516
 517         match responses[0] {
 518                 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
 519                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 520                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
 521                         check_added_monitors!(nodes[0], 1);
 522                 },
 523                 _ => panic!("Unexpected event"),
 524         }
 525         match responses[1] {
 526                 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
 527                         assert!(updates.update_add_htlcs.is_empty());
 528                         assert!(updates.update_fulfill_htlcs.is_empty());
 529                         assert!(updates.update_fail_htlcs.is_empty());
 530                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 531                         assert!(updates.update_fee.is_none());
 532                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 533
 534                         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 535                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
 536                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 537                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 538                         check_added_monitors!(nodes[0], 1);
 539                         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 540                 },
 541                 _ => panic!("Unexpected event"),
 542         }
 543
 544         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 545         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 546         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 547         check_added_monitors!(nodes[0], 0);
 548
 549         let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 550         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
 551         check_added_monitors!(nodes[1], 1);
 552
 553         expect_pending_htlcs_forwardable!(nodes[1]);
 554
 555         let events = nodes[1].node.get_and_clear_pending_events();
 556         assert_eq!(events.len(), 1);
 557         match events[0] {
 558                 Event::PaymentReceived { payment_hash, amt } => {
 559                         assert_eq!(payment_hash, our_payment_hash);
 560                         assert_eq!(amt, 1000000);
 561                 },
 562                 _ => panic!("Unexpected event"),
 563         };
 564
 565         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
 566 }
 567
 568 #[test]
 569 fn test_monitor_update_fail_no_rebroadcast() {
 570         // Tests handling of a monitor update failure when no message rebroadcasting on
 571         // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
 572         // fuzz tests.
 573         let chanmon_cfgs = create_chanmon_cfgs(2);
 574         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 575         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 576         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 577         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
 578
 579         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 580         let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
 581         nodes[0].node.send_payment(route, our_payment_hash).unwrap();
 582         check_added_monitors!(nodes[0], 1);
 583
 584         let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 585         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 586         let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
 587
 588         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 589         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
 590         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 591         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 592         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 593         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 594         check_added_monitors!(nodes[1], 1);
 595
 596         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 597         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 598         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 599         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 600         check_added_monitors!(nodes[1], 0);
 601         expect_pending_htlcs_forwardable!(nodes[1]);
 602
 603         let events = nodes[1].node.get_and_clear_pending_events();
 604         assert_eq!(events.len(), 1);
 605         match events[0] {
 606                 Event::PaymentReceived { payment_hash, .. } => {
 607                         assert_eq!(payment_hash, our_payment_hash);
 608                 },
 609                 _ => panic!("Unexpected event"),
 610         }
 611
 612         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 613 }
 614
 615 #[test]
 616 fn test_monitor_update_raa_while_paused() {
 617         // Tests handling of an RAA while monitor updating has already been marked failed.
 618         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
 619         let chanmon_cfgs = create_chanmon_cfgs(2);
 620         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 621         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 622         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 623         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
 624
 625         send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
 626
 627         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 628         let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
 629         nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
 630         check_added_monitors!(nodes[0], 1);
 631         let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 632
 633         let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 634         let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 635         nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
 636         check_added_monitors!(nodes[1], 1);
 637         let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
 638
 639         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
 640         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
 641         check_added_monitors!(nodes[1], 1);
 642         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 643
 644         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 645         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
 646         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
 647         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 648         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 649         check_added_monitors!(nodes[0], 1);
 650
 651         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
 652         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 653         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
 654         check_added_monitors!(nodes[0], 1);
 655
 656         *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
 657         let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
 658         nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
 659         check_added_monitors!(nodes[0], 0);
 660
 661         let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
 662         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
 663         check_added_monitors!(nodes[1], 1);
 664         let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
 665
 666         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
 667         check_added_monitors!(nodes[1], 1);
 668         let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
 669
 670         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
 671         check_added_monitors!(nodes[0], 1);
 672         let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 673
 674         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
 675         check_added_monitors!(nodes[0], 1);
 676         expect_pending_htlcs_forwardable!(nodes[0]);
 677         expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
 678
 679         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
 680         check_added_monitors!(nodes[1], 1);
 681         expect_pending_htlcs_forwardable!(nodes[1]);
 682         expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
 683
 684         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
 685         claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
 686 }
 687
 688 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
 689         // Tests handling of a monitor update failure when processing an incoming RAA
 690         let chanmon_cfgs = create_chanmon_cfgs(3);
 691         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 692         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 693         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 694         create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 695         let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 696
 697         // Rebalance a bit so that we can send backwards from 2 to 1.
 698         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
 699
 700         // Route a first payment that we'll fail backwards
 701         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 702
 703         // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
 704         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
 705         expect_pending_htlcs_forwardable!(nodes[2]);
 706         check_added_monitors!(nodes[2], 1);
 707
 708         let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 709         assert!(updates.update_add_htlcs.is_empty());
 710         assert!(updates.update_fulfill_htlcs.is_empty());
 711         assert_eq!(updates.update_fail_htlcs.len(), 1);
 712         assert!(updates.update_fail_malformed_htlcs.is_empty());
 713         assert!(updates.update_fee.is_none());
 714         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 715
 716         let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
 717         check_added_monitors!(nodes[0], 0);
 718
 719         // While the second channel is AwaitingRAA, forward a second payment to get it into the
 720         // holding cell.
 721         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
 722         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 723         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
 724         check_added_monitors!(nodes[0], 1);
 725
 726         let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 727         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 728         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
 729
 730         expect_pending_htlcs_forwardable!(nodes[1]);
 731         check_added_monitors!(nodes[1], 0);
 732         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 733
 734         // Now fail monitor updating.
 735         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 736         nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 737         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 738         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 739         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 740         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 741         check_added_monitors!(nodes[1], 1);
 742
 743         // Attempt to forward a third payment but fail due to the second channel being unavailable
 744         // for forwarding.
 745
 746         let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
 747         let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 748         nodes[0].node.send_payment(route, payment_hash_3).unwrap();
 749         check_added_monitors!(nodes[0], 1);
 750
 751         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
 752         send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
 753         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
 754         commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
 755         check_added_monitors!(nodes[1], 0);
 756
 757         let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
 758         assert_eq!(events_2.len(), 1);
 759         match events_2.remove(0) {
 760                 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
 761                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 762                         assert!(updates.update_fulfill_htlcs.is_empty());
 763                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 764                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 765                         assert!(updates.update_add_htlcs.is_empty());
 766                         assert!(updates.update_fee.is_none());
 767
 768                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
 769                         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
 770
 771                         let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
 772                         assert_eq!(msg_events.len(), 1);
 773                         match msg_events[0] {
 774                                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
 775                                         assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
 776                                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
 777                                 },
 778                                 _ => panic!("Unexpected event"),
 779                         }
 780
 781                         let events = nodes[0].node.get_and_clear_pending_events();
 782                         assert_eq!(events.len(), 1);
 783                         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
 784                                 assert_eq!(payment_hash, payment_hash_3);
 785                                 assert!(!rejected_by_dest);
 786                         } else { panic!("Unexpected event!"); }
 787                 },
 788                 _ => panic!("Unexpected event type!"),
 789         };
 790
 791         let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
 792                 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
 793                 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
 794                 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
 795                 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
 796                 check_added_monitors!(nodes[2], 1);
 797
 798                 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
 799                 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
 800                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
 801                 check_added_monitors!(nodes[1], 1);
 802                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 803                 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
 804                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 805                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 806                 (Some(payment_preimage_4), Some(payment_hash_4))
 807         } else { (None, None) };
 808
 809         // Restore monitor updating, ensuring we immediately get a fail-back update and a
 810         // update_add update.
 811         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
 812         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
 813         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
 814         check_added_monitors!(nodes[1], 0);
 815         expect_pending_htlcs_forwardable!(nodes[1]);
 816         check_added_monitors!(nodes[1], 1);
 817
 818         let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
 819         if test_ignore_second_cs {
 820                 assert_eq!(events_3.len(), 3);
 821         } else {
 822                 assert_eq!(events_3.len(), 2);
 823         }
 824
 825         // Note that the ordering of the events for different nodes is non-prescriptive, though the
 826         // ordering of the two events that both go to nodes[2] have to stay in the same order.
 827         let messages_a = match events_3.pop().unwrap() {
 828                 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
 829                         assert_eq!(node_id, nodes[0].node.get_our_node_id());
 830                         assert!(updates.update_fulfill_htlcs.is_empty());
 831                         assert_eq!(updates.update_fail_htlcs.len(), 1);
 832                         assert!(updates.update_fail_malformed_htlcs.is_empty());
 833                         assert!(updates.update_add_htlcs.is_empty());
 834                         assert!(updates.update_fee.is_none());
 835                         (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
 836                 },
 837                 _ => panic!("Unexpected event type!"),
 838         };
 839         let raa = if test_ignore_second_cs {
 840                 match events_3.remove(1) {
 841                         MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
 842                                 assert_eq!(node_id, nodes[2].node.get_our_node_id());
 843                                 Some(msg.clone())
 844                         },
 845                         _ => panic!("Unexpected event"),
 846                 }
 847         } else { None };
 848         let send_event_b = SendEvent::from_event(events_3.remove(0));
 849         assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
 850
 851         // Now deliver the new messages...
 852
 853         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
 854         commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
 855         let events_4 = nodes[0].node.get_and_clear_pending_events();
 856         assert_eq!(events_4.len(), 1);
 857         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
 858                 assert_eq!(payment_hash, payment_hash_1);
 859                 assert!(rejected_by_dest);
 860         } else { panic!("Unexpected event!"); }
 861
 862         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
 863         if test_ignore_second_cs {
 864                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
 865                 check_added_monitors!(nodes[2], 1);
 866                 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 867                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
 868                 check_added_monitors!(nodes[2], 1);
 869                 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 870                 assert!(bs_cs.update_add_htlcs.is_empty());
 871                 assert!(bs_cs.update_fail_htlcs.is_empty());
 872                 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
 873                 assert!(bs_cs.update_fulfill_htlcs.is_empty());
 874                 assert!(bs_cs.update_fee.is_none());
 875
 876                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
 877                 check_added_monitors!(nodes[1], 1);
 878                 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
 879                 assert!(as_cs.update_add_htlcs.is_empty());
 880                 assert!(as_cs.update_fail_htlcs.is_empty());
 881                 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
 882                 assert!(as_cs.update_fulfill_htlcs.is_empty());
 883                 assert!(as_cs.update_fee.is_none());
 884
 885                 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
 886                 check_added_monitors!(nodes[1], 1);
 887                 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
 888
 889                 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
 890                 check_added_monitors!(nodes[2], 1);
 891                 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
 892
 893                 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
 894                 check_added_monitors!(nodes[2], 1);
 895                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 896
 897                 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
 898                 check_added_monitors!(nodes[1], 1);
 899                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 900         } else {
 901                 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
 902         }
 903
 904         expect_pending_htlcs_forwardable!(nodes[2]);
 905
 906         let events_6 = nodes[2].node.get_and_clear_pending_events();
 907         assert_eq!(events_6.len(), 1);
 908         match events_6[0] {
 909                 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
 910                 _ => panic!("Unexpected event"),
 911         };
 912
 913         if test_ignore_second_cs {
 914                 expect_pending_htlcs_forwardable!(nodes[1]);
 915                 check_added_monitors!(nodes[1], 1);
 916
 917                 send_event = SendEvent::from_node(&nodes[1]);
 918                 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
 919                 assert_eq!(send_event.msgs.len(), 1);
 920                 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
 921                 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
 922
 923                 expect_pending_htlcs_forwardable!(nodes[0]);
 924
 925                 let events_9 = nodes[0].node.get_and_clear_pending_events();
 926                 assert_eq!(events_9.len(), 1);
 927                 match events_9[0] {
 928                         Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
 929                         _ => panic!("Unexpected event"),
 930                 };
 931                 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
 932         }
 933
 934         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
 935 }
 936
 937 #[test]
 938 fn test_monitor_update_fail_raa() {
 939         do_test_monitor_update_fail_raa(false);
 940         do_test_monitor_update_fail_raa(true);
 941 }
 942
 943 #[test]
 944 fn test_monitor_update_fail_reestablish() {
 945         // Simple test for message retransmission after monitor update failure on
 946         // channel_reestablish generating a monitor update (which comes from freeing holding cell
 947         // HTLCs).
 948         let chanmon_cfgs = create_chanmon_cfgs(3);
 949         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 950         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 951         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 952         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
 953         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
 954
 955         let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
 956
 957         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 958         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 959
 960         assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
 961         check_added_monitors!(nodes[2], 1);
 962         let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
 963         assert!(updates.update_add_htlcs.is_empty());
 964         assert!(updates.update_fail_htlcs.is_empty());
 965         assert!(updates.update_fail_malformed_htlcs.is_empty());
 966         assert!(updates.update_fee.is_none());
 967         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
 968         nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 969         check_added_monitors!(nodes[1], 1);
 970         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 971         commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
 972
 973         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
 974         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 975         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 976
 977         let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
 978         let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
 979
 980         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 981
 982         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 983         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 984         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
 985         check_added_monitors!(nodes[1], 1);
 986
 987         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 988         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 989
 990         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 991         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
 992
 993         assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
 994         assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
 995
 996         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
 997
 998         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
 999         check_added_monitors!(nodes[1], 0);
1000         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1001
1002         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1003         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1004         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1005         check_added_monitors!(nodes[1], 0);
1006
1007         updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1008         assert!(updates.update_add_htlcs.is_empty());
1009         assert!(updates.update_fail_htlcs.is_empty());
1010         assert!(updates.update_fail_malformed_htlcs.is_empty());
1011         assert!(updates.update_fee.is_none());
1012         assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1013         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1014         commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1015
1016         let events = nodes[0].node.get_and_clear_pending_events();
1017         assert_eq!(events.len(), 1);
1018         match events[0] {
1019                 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1020                 _ => panic!("Unexpected event"),
1021         }
1022 }
1023
1024 #[test]
1025 fn raa_no_response_awaiting_raa_state() {
1026         // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1027         // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1028         // in question (assuming it intends to respond with a CS after monitor updating is restored).
1029         // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1030         let chanmon_cfgs = create_chanmon_cfgs(2);
1031         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1032         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1033         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1034         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
1035
1036         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1037         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1038         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1039         let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1040
1041         // Queue up two payments - one will be delivered right away, one immediately goes into the
1042         // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1043         // immediately after a CS. By setting failing the monitor update failure from the CS (which
1044         // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1045         // generation during RAA while in monitor-update-failed state.
1046         nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1047         check_added_monitors!(nodes[0], 1);
1048         nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1049         check_added_monitors!(nodes[0], 0);
1050
1051         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1052         assert_eq!(events.len(), 1);
1053         let payment_event = SendEvent::from_event(events.pop().unwrap());
1054         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1055         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1056         check_added_monitors!(nodes[1], 1);
1057
1058         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1059         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1060         check_added_monitors!(nodes[0], 1);
1061         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1062         assert_eq!(events.len(), 1);
1063         let payment_event = SendEvent::from_event(events.pop().unwrap());
1064
1065         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1066         check_added_monitors!(nodes[0], 1);
1067         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1068
1069         // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1070         // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1071         // then restore channel monitor updates.
1072         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1073         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1074         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1075         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1076         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1077         check_added_monitors!(nodes[1], 1);
1078
1079         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1080         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1081         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1082         check_added_monitors!(nodes[1], 1);
1083
1084         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1085         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1086         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1087         // nodes[1] should be AwaitingRAA here!
1088         check_added_monitors!(nodes[1], 0);
1089         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1090         expect_pending_htlcs_forwardable!(nodes[1]);
1091         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1092
1093         // We send a third payment here, which is somewhat of a redundant test, but the
1094         // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1095         // commitment transaction states) whereas here we can explicitly check for it.
1096         nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1097         check_added_monitors!(nodes[0], 0);
1098         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1099
1100         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1101         check_added_monitors!(nodes[0], 1);
1102         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1103         assert_eq!(events.len(), 1);
1104         let payment_event = SendEvent::from_event(events.pop().unwrap());
1105
1106         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1107         check_added_monitors!(nodes[0], 1);
1108         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1109
1110         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1111         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1112         check_added_monitors!(nodes[1], 1);
1113         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1114
1115         // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1116         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1117         check_added_monitors!(nodes[1], 1);
1118         expect_pending_htlcs_forwardable!(nodes[1]);
1119         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1120         let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1121
1122         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1123         check_added_monitors!(nodes[0], 1);
1124
1125         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1126         check_added_monitors!(nodes[0], 1);
1127         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1128
1129         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1130         check_added_monitors!(nodes[1], 1);
1131         expect_pending_htlcs_forwardable!(nodes[1]);
1132         expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1133
1134         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1135         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1136         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1137 }
1138
1139 #[test]
1140 fn claim_while_disconnected_monitor_update_fail() {
1141         // Test for claiming a payment while disconnected and then having the resulting
1142         // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1143         // contrived case for nodes with network instability.
1144         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1145         // code introduced a regression in this test (specifically, this caught a removal of the
1146         // channel_reestablish handling ensuring the order was sensical given the messages used).
1147         let chanmon_cfgs = create_chanmon_cfgs(2);
1148         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1149         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1150         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1151         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
1152
1153         // Forward a payment for B to claim
1154         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1155
1156         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1157         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1158
1159         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1160         check_added_monitors!(nodes[1], 1);
1161
1162         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1163         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1164
1165         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1166         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1167
1168         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1169         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1170
1171         // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1172         // update.
1173         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1174
1175         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1176         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1177         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1178         check_added_monitors!(nodes[1], 1);
1179         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1180
1181         // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1182         // the monitor still failed
1183         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1184         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1185         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1186         check_added_monitors!(nodes[0], 1);
1187
1188         let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1189         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1190         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1191         check_added_monitors!(nodes[1], 1);
1192         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1193         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1194         // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1195         // until we've channel_monitor_update'd and updated for the new commitment transaction.
1196
1197         // Now un-fail the monitor, which will result in B sending its original commitment update,
1198         // receiving the commitment update from A, and the resulting commitment dances.
1199         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1200         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1201         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1202         check_added_monitors!(nodes[1], 0);
1203
1204         let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1205         assert_eq!(bs_msgs.len(), 2);
1206
1207         match bs_msgs[0] {
1208                 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1209                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1210                         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1211                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1212                         check_added_monitors!(nodes[0], 1);
1213
1214                         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1215                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1216                         check_added_monitors!(nodes[1], 1);
1217                 },
1218                 _ => panic!("Unexpected event"),
1219         }
1220
1221         match bs_msgs[1] {
1222                 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1223                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1224                         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1225                         check_added_monitors!(nodes[0], 1);
1226                 },
1227                 _ => panic!("Unexpected event"),
1228         }
1229
1230         let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1231
1232         let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1233         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1234         check_added_monitors!(nodes[0], 1);
1235         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1236
1237         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1238         check_added_monitors!(nodes[1], 1);
1239         let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1240         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1241         check_added_monitors!(nodes[1], 1);
1242
1243         expect_pending_htlcs_forwardable!(nodes[1]);
1244         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1245
1246         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1247         check_added_monitors!(nodes[0], 1);
1248
1249         let events = nodes[0].node.get_and_clear_pending_events();
1250         assert_eq!(events.len(), 1);
1251         match events[0] {
1252                 Event::PaymentSent { ref payment_preimage } => {
1253                         assert_eq!(*payment_preimage, payment_preimage_1);
1254                 },
1255                 _ => panic!("Unexpected event"),
1256         }
1257
1258         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1259 }
1260
1261 #[test]
1262 fn monitor_failed_no_reestablish_response() {
1263         // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1264         // response to a commitment_signed.
1265         // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1266         // debug_assert!() failure in channel_reestablish handling.
1267         let chanmon_cfgs = create_chanmon_cfgs(2);
1268         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1269         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1270         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1271         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
1272
1273         // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1274         // on receipt).
1275         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1276         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1277         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1278         check_added_monitors!(nodes[0], 1);
1279
1280         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1281         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1282         assert_eq!(events.len(), 1);
1283         let payment_event = SendEvent::from_event(events.pop().unwrap());
1284         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1285         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1286         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1287         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1288         check_added_monitors!(nodes[1], 1);
1289
1290         // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1291         // is still failing to update monitors.
1292         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1293         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1294
1295         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1296         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1297
1298         let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1299         let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1300
1301         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1302         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1303
1304         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1305         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1306         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1307         check_added_monitors!(nodes[1], 0);
1308         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1309
1310         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1311         check_added_monitors!(nodes[0], 1);
1312         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1313         check_added_monitors!(nodes[0], 1);
1314
1315         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1316         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1317         check_added_monitors!(nodes[1], 1);
1318
1319         expect_pending_htlcs_forwardable!(nodes[1]);
1320         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1321
1322         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1323 }
1324
1325 #[test]
1326 fn first_message_on_recv_ordering() {
1327         // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1328         // messages, we're willing to flip the order of response messages if neccessary in resposne to
1329         // a commitment_signed which needs to send an RAA first.
1330         // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1331         // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1332         // response. To do this, we start routing two payments, with the final RAA for the first being
1333         // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1334         // have no pending response but will want to send a RAA/CS (with the updates for the second
1335         // payment applied).
1336         // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1337         let chanmon_cfgs = create_chanmon_cfgs(2);
1338         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1339         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1340         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1341         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
1342
1343         // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1344         // can deliver it and fail the monitor update.
1345         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1346         let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1347         nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1348         check_added_monitors!(nodes[0], 1);
1349
1350         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1351         assert_eq!(events.len(), 1);
1352         let payment_event = SendEvent::from_event(events.pop().unwrap());
1353         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1354         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1355         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1356         check_added_monitors!(nodes[1], 1);
1357         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1358
1359         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1360         check_added_monitors!(nodes[0], 1);
1361         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1362         check_added_monitors!(nodes[0], 1);
1363
1364         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1365
1366         // Route the second payment, generating an update_add_htlc/commitment_signed
1367         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1368         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1369         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1370         check_added_monitors!(nodes[0], 1);
1371         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1372         assert_eq!(events.len(), 1);
1373         let payment_event = SendEvent::from_event(events.pop().unwrap());
1374         assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1375
1376         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1377
1378         // Deliver the final RAA for the first payment, which does not require a response. RAAs
1379         // generally require a commitment_signed, so the fact that we're expecting an opposite response
1380         // to the next message also tests resetting the delivery order.
1381         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1382         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1383         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1384         check_added_monitors!(nodes[1], 1);
1385
1386         // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1387         // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1388         // appropriate HTLC acceptance).
1389         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1390         nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1391         check_added_monitors!(nodes[1], 1);
1392         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1393         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1394
1395         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1396         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1397         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1398         check_added_monitors!(nodes[1], 0);
1399
1400         expect_pending_htlcs_forwardable!(nodes[1]);
1401         expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1402
1403         let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1404         nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1405         check_added_monitors!(nodes[0], 1);
1406         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1407         check_added_monitors!(nodes[0], 1);
1408
1409         let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1410         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1411         check_added_monitors!(nodes[1], 1);
1412
1413         expect_pending_htlcs_forwardable!(nodes[1]);
1414         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1415
1416         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1417         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1418 }
1419
1420 #[test]
1421 fn test_monitor_update_fail_claim() {
1422         // Basic test for monitor update failures when processing claim_funds calls.
1423         // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1424         // update to claim the payment. We then send a payment C->B->A, making the forward of this
1425         // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1426         // updating and claim the payment on B.
1427         let chanmon_cfgs = create_chanmon_cfgs(3);
1428         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1429         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1430         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1431         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1432         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1433
1434         // Rebalance a bit so that we can send backwards from 3 to 2.
1435         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1436
1437         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1438
1439         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1440         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1441         check_added_monitors!(nodes[1], 1);
1442
1443         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1444         let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1445         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1446         check_added_monitors!(nodes[2], 1);
1447
1448         // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1449         // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1450         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1451
1452         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1453         assert_eq!(events.len(), 1);
1454         let payment_event = SendEvent::from_event(events.pop().unwrap());
1455         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1456         let events = nodes[1].node.get_and_clear_pending_msg_events();
1457         assert_eq!(events.len(), 0);
1458         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1459         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1460
1461         let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1462         nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1463         commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1464
1465         let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1466         assert_eq!(msg_events.len(), 1);
1467         match msg_events[0] {
1468                 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1469                         assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1470                         assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1471                 },
1472                 _ => panic!("Unexpected event"),
1473         }
1474
1475         let events = nodes[2].node.get_and_clear_pending_events();
1476         assert_eq!(events.len(), 1);
1477         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1478                 assert_eq!(payment_hash, payment_hash_2);
1479                 assert!(!rejected_by_dest);
1480         } else { panic!("Unexpected event!"); }
1481
1482         // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1483         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1484         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1485         check_added_monitors!(nodes[1], 0);
1486
1487         let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1488         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1489         commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1490
1491         let events = nodes[0].node.get_and_clear_pending_events();
1492         assert_eq!(events.len(), 1);
1493         if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1494                 assert_eq!(payment_preimage, payment_preimage_1);
1495         } else { panic!("Unexpected event!"); }
1496 }
1497
1498 #[test]
1499 fn test_monitor_update_on_pending_forwards() {
1500         // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1501         // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1502         // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1503         // from C to A will be pending a forward to A.
1504         let chanmon_cfgs = create_chanmon_cfgs(3);
1505         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1506         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1507         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1508         let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1509         create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1510
1511         // Rebalance a bit so that we can send backwards from 3 to 1.
1512         send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1513
1514         let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1515         assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1516         expect_pending_htlcs_forwardable!(nodes[2]);
1517         check_added_monitors!(nodes[2], 1);
1518
1519         let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1520         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1521         commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1522         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1523
1524         let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1525         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1526         nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1527         check_added_monitors!(nodes[2], 1);
1528
1529         let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1530         assert_eq!(events.len(), 1);
1531         let payment_event = SendEvent::from_event(events.pop().unwrap());
1532         nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1533         commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1534
1535         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1536         expect_pending_htlcs_forwardable!(nodes[1]);
1537         check_added_monitors!(nodes[1], 1);
1538         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1539         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1540
1541         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1542         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1543         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1544         check_added_monitors!(nodes[1], 0);
1545
1546         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1547         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1548         nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1549         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1550
1551         let events = nodes[0].node.get_and_clear_pending_events();
1552         assert_eq!(events.len(), 2);
1553         if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1554                 assert_eq!(payment_hash, payment_hash_1);
1555                 assert!(rejected_by_dest);
1556         } else { panic!("Unexpected event!"); }
1557         match events[1] {
1558                 Event::PendingHTLCsForwardable { .. } => { },
1559                 _ => panic!("Unexpected event"),
1560         };
1561         nodes[0].node.process_pending_htlc_forwards();
1562         expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1563
1564         claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1565 }
1566
1567 #[test]
1568 fn monitor_update_claim_fail_no_response() {
1569         // Test for claim_funds resulting in both a monitor update failure and no message response (due
1570         // to channel being AwaitingRAA).
1571         // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1572         // code was broken.
1573         let chanmon_cfgs = create_chanmon_cfgs(2);
1574         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1575         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1576         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1577         let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported()).2;
1578
1579         // Forward a payment for B to claim
1580         let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1581
1582         // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1583         let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1584         let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1585         nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1586         check_added_monitors!(nodes[0], 1);
1587
1588         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1589         assert_eq!(events.len(), 1);
1590         let payment_event = SendEvent::from_event(events.pop().unwrap());
1591         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1592         let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1593
1594         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1595         assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1596         check_added_monitors!(nodes[1], 1);
1597         let events = nodes[1].node.get_and_clear_pending_msg_events();
1598         assert_eq!(events.len(), 0);
1599         nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1600
1601         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1602         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1603         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1604         check_added_monitors!(nodes[1], 0);
1605         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1606
1607         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1608         check_added_monitors!(nodes[1], 1);
1609         expect_pending_htlcs_forwardable!(nodes[1]);
1610         expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1611
1612         let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1613         nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1614         commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1615
1616         let events = nodes[0].node.get_and_clear_pending_events();
1617         assert_eq!(events.len(), 1);
1618         match events[0] {
1619                 Event::PaymentSent { ref payment_preimage } => {
1620                         assert_eq!(*payment_preimage, payment_preimage_1);
1621                 },
1622                 _ => panic!("Unexpected event"),
1623         }
1624
1625         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1626 }
1627
1628 // Note that restore_between_fails with !fail_on_generate is useless
1629 // Also note that !fail_on_generate && !fail_on_signed is useless
1630 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1631 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1632 // restore_b_before_conf has no meaning if !confirm_a_first
1633 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1634         // Test that if the monitor update generated by funding_transaction_generated fails we continue
1635         // the channel setup happily after the update is restored.
1636         let chanmon_cfgs = create_chanmon_cfgs(2);
1637         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1638         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1639         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1640
1641         nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1642         nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1643         nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1644
1645         let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1646
1647         if fail_on_generate {
1648                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1649         }
1650         nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1651         check_added_monitors!(nodes[0], 1);
1652
1653         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1654         let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1655         let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1656         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1657         check_added_monitors!(nodes[1], 1);
1658
1659         if restore_between_fails {
1660                 assert!(fail_on_generate);
1661                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1662                 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1663                 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1664                 check_added_monitors!(nodes[0], 0);
1665                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1666                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1667         }
1668
1669         if fail_on_signed {
1670                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1671         } else {
1672                 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1673                 assert!(fail_on_generate); // Somebody has to fail
1674         }
1675         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1676         if fail_on_signed || !restore_between_fails {
1677                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1678                 if fail_on_generate && !restore_between_fails {
1679                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1680                         check_added_monitors!(nodes[0], 1);
1681                 } else {
1682                         nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1683                         check_added_monitors!(nodes[0], 1);
1684                 }
1685                 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1686                 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1687                 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1688                 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1689                 check_added_monitors!(nodes[0], 0);
1690         } else {
1691                 check_added_monitors!(nodes[0], 1);
1692         }
1693
1694         let events = nodes[0].node.get_and_clear_pending_events();
1695         assert_eq!(events.len(), 1);
1696         match events[0] {
1697                 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1698                         assert_eq!(user_channel_id, 43);
1699                         assert_eq!(*funding_txo, funding_output);
1700                 },
1701                 _ => panic!("Unexpected event"),
1702         };
1703
1704         if confirm_a_first {
1705                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1706                 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1707         } else {
1708                 assert!(!restore_b_before_conf);
1709                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1710                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1711         }
1712
1713         // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1714         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1715         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1716         reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1717         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1718         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1719
1720         if !restore_b_before_conf {
1721                 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1722                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1723                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1724         }
1725
1726         *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1727         let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1728         nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1729         check_added_monitors!(nodes[1], 0);
1730
1731         let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1732                 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1733
1734                 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1735                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1736                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1737         } else {
1738                 if restore_b_before_conf {
1739                         confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1740                 }
1741                 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1742                 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1743         };
1744         for node in nodes.iter() {
1745                 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1746                 node.router.handle_channel_update(&as_update).unwrap();
1747                 node.router.handle_channel_update(&bs_update).unwrap();
1748         }
1749
1750         send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1751         close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1752 }
1753
1754 #[test]
1755 fn during_funding_monitor_fail() {
1756         do_during_funding_monitor_fail(false, false, true, true, true);
1757         do_during_funding_monitor_fail(true, false, true, false, false);
1758         do_during_funding_monitor_fail(true, true, true, true, false);
1759         do_during_funding_monitor_fail(true, true, false, false, false);
1760 }