1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
4 //! here. See also the chanmon_fail_consistency fuzz test.
6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
7 use ln::channelmonitor::ChannelMonitorUpdateErr;
8 use ln::features::InitFeatures;
10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
12 use util::errors::APIError;
14 use bitcoin_hashes::sha256::Hash as Sha256;
15 use bitcoin_hashes::Hash;
17 use ln::functional_test_utils::*;
20 fn test_simple_monitor_permanent_update_fail() {
21 // Test that we handle a simple permanent monitor update failure
22 let chanmon_cfgs = create_chanmon_cfgs(2);
23 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
24 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
25 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
26 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
28 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
29 let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
31 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
32 if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
33 check_added_monitors!(nodes[0], 1);
35 let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
36 assert_eq!(events_1.len(), 2);
38 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
39 _ => panic!("Unexpected event"),
42 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
43 _ => panic!("Unexpected event"),
46 // TODO: Once we hit the chain with the failure transaction we should check that we get a
47 // PaymentFailed event
49 assert_eq!(nodes[0].node.list_channels().len(), 0);
52 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
53 // Test that we can recover from a simple temporary monitor update failure optionally with
54 // a disconnect in between
55 let chanmon_cfgs = create_chanmon_cfgs(2);
56 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
57 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
58 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
59 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
61 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
62 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
64 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
65 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
66 check_added_monitors!(nodes[0], 1);
68 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
69 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
70 assert_eq!(nodes[0].node.list_channels().len(), 1);
73 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
74 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
75 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
78 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
79 nodes[0].node.test_restore_channel_monitor();
80 check_added_monitors!(nodes[0], 1);
82 let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
83 assert_eq!(events_2.len(), 1);
84 let payment_event = SendEvent::from_event(events_2.pop().unwrap());
85 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
86 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
87 commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
89 expect_pending_htlcs_forwardable!(nodes[1]);
91 let events_3 = nodes[1].node.get_and_clear_pending_events();
92 assert_eq!(events_3.len(), 1);
94 Event::PaymentReceived { ref payment_hash, amt } => {
95 assert_eq!(payment_hash_1, *payment_hash);
96 assert_eq!(amt, 1000000);
98 _ => panic!("Unexpected event"),
101 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
103 // Now set it to failed again...
104 let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
105 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
106 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
107 check_added_monitors!(nodes[0], 1);
109 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
110 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
111 assert_eq!(nodes[0].node.list_channels().len(), 1);
114 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
115 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
116 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
119 // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
120 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
121 nodes[0].node.test_restore_channel_monitor();
122 check_added_monitors!(nodes[0], 1);
123 check_closed_broadcast!(nodes[0], false);
125 // TODO: Once we hit the chain with the failure transaction we should check that we get a
126 // PaymentFailed event
128 assert_eq!(nodes[0].node.list_channels().len(), 0);
132 fn test_simple_monitor_temporary_update_fail() {
133 do_test_simple_monitor_temporary_update_fail(false);
134 do_test_simple_monitor_temporary_update_fail(true);
137 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
138 let disconnect_flags = 8 | 16;
140 // Test that we can recover from a temporary monitor update failure with some in-flight
141 // HTLCs going on at the same time potentially with some disconnection thrown in.
142 // * First we route a payment, then get a temporary monitor update failure when trying to
143 // route a second payment. We then claim the first payment.
144 // * If disconnect_count is set, we will disconnect at this point (which is likely as
145 // TemporaryFailure likely indicates net disconnect which resulted in failing to update
146 // the ChannelMonitor on a watchtower).
147 // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
148 // immediately, otherwise we wait disconnect and deliver them via the reconnect
149 // channel_reestablish processing (ie disconnect_count & 16 makes no sense if
150 // disconnect_count & !disconnect_flags is 0).
151 // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
152 // through message sending, potentially disconnect/reconnecting multiple times based on
153 // disconnect_count, to get the update_fulfill_htlc through.
154 // * We then walk through more message exchanges to get the original update_add_htlc
155 // through, swapping message ordering based on disconnect_count & 8 and optionally
156 // disconnect/reconnecting based on disconnect_count.
157 let chanmon_cfgs = create_chanmon_cfgs(2);
158 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
159 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
160 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
161 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
163 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
165 // Now try to send a second payment which will fail to send
166 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
167 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
169 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
170 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
171 check_added_monitors!(nodes[0], 1);
173 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
174 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
175 assert_eq!(nodes[0].node.list_channels().len(), 1);
177 // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
178 // but nodes[0] won't respond since it is frozen.
179 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
180 check_added_monitors!(nodes[1], 1);
181 let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
182 assert_eq!(events_2.len(), 1);
183 let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
184 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
185 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
186 assert!(update_add_htlcs.is_empty());
187 assert_eq!(update_fulfill_htlcs.len(), 1);
188 assert!(update_fail_htlcs.is_empty());
189 assert!(update_fail_malformed_htlcs.is_empty());
190 assert!(update_fee.is_none());
192 if (disconnect_count & 16) == 0 {
193 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
194 let events_3 = nodes[0].node.get_and_clear_pending_events();
195 assert_eq!(events_3.len(), 1);
197 Event::PaymentSent { ref payment_preimage } => {
198 assert_eq!(*payment_preimage, payment_preimage_1);
200 _ => panic!("Unexpected event"),
203 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
204 check_added_monitors!(nodes[0], 1);
205 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
206 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
209 (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
211 _ => panic!("Unexpected event"),
214 if disconnect_count & !disconnect_flags > 0 {
215 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
216 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
219 // Now fix monitor updating...
220 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
221 nodes[0].node.test_restore_channel_monitor();
222 check_added_monitors!(nodes[0], 1);
224 macro_rules! disconnect_reconnect_peers { () => { {
225 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
226 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
228 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
229 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
230 assert_eq!(reestablish_1.len(), 1);
231 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
232 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
233 assert_eq!(reestablish_2.len(), 1);
235 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
236 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
237 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
238 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
240 assert!(as_resp.0.is_none());
241 assert!(bs_resp.0.is_none());
243 (reestablish_1, reestablish_2, as_resp, bs_resp)
246 let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
247 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
248 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
250 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
251 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
252 assert_eq!(reestablish_1.len(), 1);
253 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
254 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
255 assert_eq!(reestablish_2.len(), 1);
257 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
258 check_added_monitors!(nodes[0], 0);
259 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
260 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
261 check_added_monitors!(nodes[1], 0);
262 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
264 assert!(as_resp.0.is_none());
265 assert!(bs_resp.0.is_none());
267 assert!(bs_resp.1.is_none());
268 if (disconnect_count & 16) == 0 {
269 assert!(bs_resp.2.is_none());
271 assert!(as_resp.1.is_some());
272 assert!(as_resp.2.is_some());
273 assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
275 assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
276 assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
277 assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
278 assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
279 assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
280 assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
282 assert!(as_resp.1.is_none());
284 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
285 let events_3 = nodes[0].node.get_and_clear_pending_events();
286 assert_eq!(events_3.len(), 1);
288 Event::PaymentSent { ref payment_preimage } => {
289 assert_eq!(*payment_preimage, payment_preimage_1);
291 _ => panic!("Unexpected event"),
294 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
295 let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
296 // No commitment_signed so get_event_msg's assert(len == 1) passes
297 check_added_monitors!(nodes[0], 1);
299 as_resp.1 = Some(as_resp_raa);
303 if disconnect_count & !disconnect_flags > 1 {
304 let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
306 if (disconnect_count & 16) == 0 {
307 assert!(reestablish_1 == second_reestablish_1);
308 assert!(reestablish_2 == second_reestablish_2);
310 assert!(as_resp == second_as_resp);
311 assert!(bs_resp == second_bs_resp);
314 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
316 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
317 assert_eq!(events_4.len(), 2);
318 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
319 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
320 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
323 _ => panic!("Unexpected event"),
327 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
329 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
330 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
331 let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
332 // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
333 check_added_monitors!(nodes[1], 1);
335 if disconnect_count & !disconnect_flags > 2 {
336 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
338 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
339 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
341 assert!(as_resp.2.is_none());
342 assert!(bs_resp.2.is_none());
345 let as_commitment_update;
346 let bs_second_commitment_update;
348 macro_rules! handle_bs_raa { () => {
349 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
350 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
351 assert!(as_commitment_update.update_add_htlcs.is_empty());
352 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
353 assert!(as_commitment_update.update_fail_htlcs.is_empty());
354 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
355 assert!(as_commitment_update.update_fee.is_none());
356 check_added_monitors!(nodes[0], 1);
359 macro_rules! handle_initial_raa { () => {
360 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
361 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
362 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
363 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
364 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
365 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
366 assert!(bs_second_commitment_update.update_fee.is_none());
367 check_added_monitors!(nodes[1], 1);
370 if (disconnect_count & 8) == 0 {
373 if disconnect_count & !disconnect_flags > 3 {
374 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
376 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
377 assert!(bs_resp.1.is_none());
379 assert!(as_resp.2.unwrap() == as_commitment_update);
380 assert!(bs_resp.2.is_none());
382 assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
385 handle_initial_raa!();
387 if disconnect_count & !disconnect_flags > 4 {
388 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
390 assert!(as_resp.1.is_none());
391 assert!(bs_resp.1.is_none());
393 assert!(as_resp.2.unwrap() == as_commitment_update);
394 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
397 handle_initial_raa!();
399 if disconnect_count & !disconnect_flags > 3 {
400 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
402 assert!(as_resp.1.is_none());
403 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
405 assert!(as_resp.2.is_none());
406 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
408 assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
413 if disconnect_count & !disconnect_flags > 4 {
414 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
416 assert!(as_resp.1.is_none());
417 assert!(bs_resp.1.is_none());
419 assert!(as_resp.2.unwrap() == as_commitment_update);
420 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
424 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
425 let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
426 // No commitment_signed so get_event_msg's assert(len == 1) passes
427 check_added_monitors!(nodes[0], 1);
429 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
430 let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
431 // No commitment_signed so get_event_msg's assert(len == 1) passes
432 check_added_monitors!(nodes[1], 1);
434 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
435 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
436 check_added_monitors!(nodes[1], 1);
438 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
439 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
440 check_added_monitors!(nodes[0], 1);
442 expect_pending_htlcs_forwardable!(nodes[1]);
444 let events_5 = nodes[1].node.get_and_clear_pending_events();
445 assert_eq!(events_5.len(), 1);
447 Event::PaymentReceived { ref payment_hash, amt } => {
448 assert_eq!(payment_hash_2, *payment_hash);
449 assert_eq!(amt, 1000000);
451 _ => panic!("Unexpected event"),
454 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
458 fn test_monitor_temporary_update_fail_a() {
459 do_test_monitor_temporary_update_fail(0);
460 do_test_monitor_temporary_update_fail(1);
461 do_test_monitor_temporary_update_fail(2);
462 do_test_monitor_temporary_update_fail(3);
463 do_test_monitor_temporary_update_fail(4);
464 do_test_monitor_temporary_update_fail(5);
468 fn test_monitor_temporary_update_fail_b() {
469 do_test_monitor_temporary_update_fail(2 | 8);
470 do_test_monitor_temporary_update_fail(3 | 8);
471 do_test_monitor_temporary_update_fail(4 | 8);
472 do_test_monitor_temporary_update_fail(5 | 8);
476 fn test_monitor_temporary_update_fail_c() {
477 do_test_monitor_temporary_update_fail(1 | 16);
478 do_test_monitor_temporary_update_fail(2 | 16);
479 do_test_monitor_temporary_update_fail(3 | 16);
480 do_test_monitor_temporary_update_fail(2 | 8 | 16);
481 do_test_monitor_temporary_update_fail(3 | 8 | 16);
485 fn test_monitor_update_fail_cs() {
486 // Tests handling of a monitor update failure when processing an incoming commitment_signed
487 let chanmon_cfgs = create_chanmon_cfgs(2);
488 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
489 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
490 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
491 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
493 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
494 let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
495 nodes[0].node.send_payment(route, our_payment_hash).unwrap();
496 check_added_monitors!(nodes[0], 1);
498 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
499 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
501 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
502 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
503 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
504 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
505 check_added_monitors!(nodes[1], 1);
506 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
508 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
509 nodes[1].node.test_restore_channel_monitor();
510 check_added_monitors!(nodes[1], 1);
511 let responses = nodes[1].node.get_and_clear_pending_msg_events();
512 assert_eq!(responses.len(), 2);
515 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
516 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
517 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
518 check_added_monitors!(nodes[0], 1);
520 _ => panic!("Unexpected event"),
523 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
524 assert!(updates.update_add_htlcs.is_empty());
525 assert!(updates.update_fulfill_htlcs.is_empty());
526 assert!(updates.update_fail_htlcs.is_empty());
527 assert!(updates.update_fail_malformed_htlcs.is_empty());
528 assert!(updates.update_fee.is_none());
529 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
531 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
532 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
533 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
534 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
535 check_added_monitors!(nodes[0], 1);
536 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
538 _ => panic!("Unexpected event"),
541 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
542 nodes[0].node.test_restore_channel_monitor();
543 check_added_monitors!(nodes[0], 1);
545 let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
546 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
547 check_added_monitors!(nodes[1], 1);
549 expect_pending_htlcs_forwardable!(nodes[1]);
551 let events = nodes[1].node.get_and_clear_pending_events();
552 assert_eq!(events.len(), 1);
554 Event::PaymentReceived { payment_hash, amt } => {
555 assert_eq!(payment_hash, our_payment_hash);
556 assert_eq!(amt, 1000000);
558 _ => panic!("Unexpected event"),
561 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
565 fn test_monitor_update_fail_no_rebroadcast() {
566 // Tests handling of a monitor update failure when no message rebroadcasting on
567 // test_restore_channel_monitor() is required. Backported from
568 // chanmon_fail_consistency fuzz tests.
569 let chanmon_cfgs = create_chanmon_cfgs(2);
570 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
571 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
572 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
573 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
575 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
576 let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
577 nodes[0].node.send_payment(route, our_payment_hash).unwrap();
578 check_added_monitors!(nodes[0], 1);
580 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
581 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
582 let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
584 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
585 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
586 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
587 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
588 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
589 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
590 check_added_monitors!(nodes[1], 1);
592 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
593 nodes[1].node.test_restore_channel_monitor();
594 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
595 check_added_monitors!(nodes[1], 1);
596 expect_pending_htlcs_forwardable!(nodes[1]);
598 let events = nodes[1].node.get_and_clear_pending_events();
599 assert_eq!(events.len(), 1);
601 Event::PaymentReceived { payment_hash, .. } => {
602 assert_eq!(payment_hash, our_payment_hash);
604 _ => panic!("Unexpected event"),
607 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
611 fn test_monitor_update_raa_while_paused() {
612 // Tests handling of an RAA while monitor updating has already been marked failed.
613 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
614 let chanmon_cfgs = create_chanmon_cfgs(2);
615 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
616 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
617 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
618 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
620 send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
622 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
623 let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
624 nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
625 check_added_monitors!(nodes[0], 1);
626 let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
628 let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
629 let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
630 nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
631 check_added_monitors!(nodes[1], 1);
632 let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
634 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
635 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
636 check_added_monitors!(nodes[1], 1);
637 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
639 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
640 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
641 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
642 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
643 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
644 check_added_monitors!(nodes[0], 1);
646 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
647 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
648 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
649 check_added_monitors!(nodes[0], 1);
651 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
652 nodes[0].node.test_restore_channel_monitor();
653 check_added_monitors!(nodes[0], 1);
655 let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
656 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
657 check_added_monitors!(nodes[1], 1);
658 let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
660 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
661 check_added_monitors!(nodes[1], 1);
662 let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
664 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
665 check_added_monitors!(nodes[0], 1);
666 let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
668 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
669 check_added_monitors!(nodes[0], 1);
670 expect_pending_htlcs_forwardable!(nodes[0]);
671 expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
673 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
674 check_added_monitors!(nodes[1], 1);
675 expect_pending_htlcs_forwardable!(nodes[1]);
676 expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
678 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
679 claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
682 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
683 // Tests handling of a monitor update failure when processing an incoming RAA
684 let chanmon_cfgs = create_chanmon_cfgs(3);
685 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
686 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
687 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
688 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
689 let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
691 // Rebalance a bit so that we can send backwards from 2 to 1.
692 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
694 // Route a first payment that we'll fail backwards
695 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
697 // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
698 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
699 expect_pending_htlcs_forwardable!(nodes[2]);
700 check_added_monitors!(nodes[2], 1);
702 let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
703 assert!(updates.update_add_htlcs.is_empty());
704 assert!(updates.update_fulfill_htlcs.is_empty());
705 assert_eq!(updates.update_fail_htlcs.len(), 1);
706 assert!(updates.update_fail_malformed_htlcs.is_empty());
707 assert!(updates.update_fee.is_none());
708 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
710 let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
711 check_added_monitors!(nodes[0], 0);
713 // While the second channel is AwaitingRAA, forward a second payment to get it into the
715 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
716 let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
717 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
718 check_added_monitors!(nodes[0], 1);
720 let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
721 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
722 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
724 expect_pending_htlcs_forwardable!(nodes[1]);
725 check_added_monitors!(nodes[1], 0);
726 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
728 // Now fail monitor updating.
729 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
730 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
731 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
732 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
733 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
734 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
735 check_added_monitors!(nodes[1], 1);
737 // Attempt to forward a third payment but fail due to the second channel being unavailable
740 let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
741 let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
742 nodes[0].node.send_payment(route, payment_hash_3).unwrap();
743 check_added_monitors!(nodes[0], 1);
745 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
746 send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
747 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
748 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
749 check_added_monitors!(nodes[1], 0);
751 let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
752 assert_eq!(events_2.len(), 1);
753 match events_2.remove(0) {
754 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
755 assert_eq!(node_id, nodes[0].node.get_our_node_id());
756 assert!(updates.update_fulfill_htlcs.is_empty());
757 assert_eq!(updates.update_fail_htlcs.len(), 1);
758 assert!(updates.update_fail_malformed_htlcs.is_empty());
759 assert!(updates.update_add_htlcs.is_empty());
760 assert!(updates.update_fee.is_none());
762 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
763 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
765 let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
766 assert_eq!(msg_events.len(), 1);
767 match msg_events[0] {
768 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
769 assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
770 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
772 _ => panic!("Unexpected event"),
775 let events = nodes[0].node.get_and_clear_pending_events();
776 assert_eq!(events.len(), 1);
777 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
778 assert_eq!(payment_hash, payment_hash_3);
779 assert!(!rejected_by_dest);
780 } else { panic!("Unexpected event!"); }
782 _ => panic!("Unexpected event type!"),
785 let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
786 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
787 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
788 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
789 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
790 check_added_monitors!(nodes[2], 1);
792 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
793 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
794 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
795 check_added_monitors!(nodes[1], 1);
796 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
797 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
798 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
799 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
800 (Some(payment_preimage_4), Some(payment_hash_4))
801 } else { (None, None) };
803 // Restore monitor updating, ensuring we immediately get a fail-back update and a
804 // update_add update.
805 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
806 nodes[1].node.test_restore_channel_monitor();
807 check_added_monitors!(nodes[1], 1);
808 expect_pending_htlcs_forwardable!(nodes[1]);
809 check_added_monitors!(nodes[1], 1);
811 let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
812 if test_ignore_second_cs {
813 assert_eq!(events_3.len(), 3);
815 assert_eq!(events_3.len(), 2);
818 // Note that the ordering of the events for different nodes is non-prescriptive, though the
819 // ordering of the two events that both go to nodes[2] have to stay in the same order.
820 let messages_a = match events_3.pop().unwrap() {
821 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
822 assert_eq!(node_id, nodes[0].node.get_our_node_id());
823 assert!(updates.update_fulfill_htlcs.is_empty());
824 assert_eq!(updates.update_fail_htlcs.len(), 1);
825 assert!(updates.update_fail_malformed_htlcs.is_empty());
826 assert!(updates.update_add_htlcs.is_empty());
827 assert!(updates.update_fee.is_none());
828 (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
830 _ => panic!("Unexpected event type!"),
832 let raa = if test_ignore_second_cs {
833 match events_3.remove(1) {
834 MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
835 assert_eq!(node_id, nodes[2].node.get_our_node_id());
838 _ => panic!("Unexpected event"),
841 let send_event_b = SendEvent::from_event(events_3.remove(0));
842 assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
844 // Now deliver the new messages...
846 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
847 commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
848 let events_4 = nodes[0].node.get_and_clear_pending_events();
849 assert_eq!(events_4.len(), 1);
850 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
851 assert_eq!(payment_hash, payment_hash_1);
852 assert!(rejected_by_dest);
853 } else { panic!("Unexpected event!"); }
855 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
856 if test_ignore_second_cs {
857 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
858 check_added_monitors!(nodes[2], 1);
859 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
860 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
861 check_added_monitors!(nodes[2], 1);
862 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
863 assert!(bs_cs.update_add_htlcs.is_empty());
864 assert!(bs_cs.update_fail_htlcs.is_empty());
865 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
866 assert!(bs_cs.update_fulfill_htlcs.is_empty());
867 assert!(bs_cs.update_fee.is_none());
869 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
870 check_added_monitors!(nodes[1], 1);
871 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
872 assert!(as_cs.update_add_htlcs.is_empty());
873 assert!(as_cs.update_fail_htlcs.is_empty());
874 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
875 assert!(as_cs.update_fulfill_htlcs.is_empty());
876 assert!(as_cs.update_fee.is_none());
878 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
879 check_added_monitors!(nodes[1], 1);
880 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
882 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
883 check_added_monitors!(nodes[2], 1);
884 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
886 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
887 check_added_monitors!(nodes[2], 1);
888 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
890 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
891 check_added_monitors!(nodes[1], 1);
892 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
894 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
897 expect_pending_htlcs_forwardable!(nodes[2]);
899 let events_6 = nodes[2].node.get_and_clear_pending_events();
900 assert_eq!(events_6.len(), 1);
902 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
903 _ => panic!("Unexpected event"),
906 if test_ignore_second_cs {
907 expect_pending_htlcs_forwardable!(nodes[1]);
908 check_added_monitors!(nodes[1], 1);
910 send_event = SendEvent::from_node(&nodes[1]);
911 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
912 assert_eq!(send_event.msgs.len(), 1);
913 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
914 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
916 expect_pending_htlcs_forwardable!(nodes[0]);
918 let events_9 = nodes[0].node.get_and_clear_pending_events();
919 assert_eq!(events_9.len(), 1);
921 Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
922 _ => panic!("Unexpected event"),
924 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
927 claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
931 fn test_monitor_update_fail_raa() {
932 do_test_monitor_update_fail_raa(false);
933 do_test_monitor_update_fail_raa(true);
937 fn test_monitor_update_fail_reestablish() {
938 // Simple test for message retransmission after monitor update failure on
939 // channel_reestablish generating a monitor update (which comes from freeing holding cell
941 let chanmon_cfgs = create_chanmon_cfgs(3);
942 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
943 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
944 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
945 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
946 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
948 let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
950 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
951 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
953 assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
954 check_added_monitors!(nodes[2], 1);
955 let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
956 assert!(updates.update_add_htlcs.is_empty());
957 assert!(updates.update_fail_htlcs.is_empty());
958 assert!(updates.update_fail_malformed_htlcs.is_empty());
959 assert!(updates.update_fee.is_none());
960 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
961 nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
962 check_added_monitors!(nodes[1], 1);
963 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
964 commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
966 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
967 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
968 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
970 let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
971 let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
973 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
975 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
976 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
977 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
978 check_added_monitors!(nodes[1], 1);
980 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
981 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
983 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
984 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
986 assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
987 assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
989 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
991 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
992 check_added_monitors!(nodes[1], 0);
993 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
995 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
996 nodes[1].node.test_restore_channel_monitor();
997 check_added_monitors!(nodes[1], 1);
999 updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1000 assert!(updates.update_add_htlcs.is_empty());
1001 assert!(updates.update_fail_htlcs.is_empty());
1002 assert!(updates.update_fail_malformed_htlcs.is_empty());
1003 assert!(updates.update_fee.is_none());
1004 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1005 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1006 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1008 let events = nodes[0].node.get_and_clear_pending_events();
1009 assert_eq!(events.len(), 1);
1011 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1012 _ => panic!("Unexpected event"),
1017 fn raa_no_response_awaiting_raa_state() {
1018 // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1019 // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1020 // in question (assuming it intends to respond with a CS after monitor updating is restored).
1021 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1022 let chanmon_cfgs = create_chanmon_cfgs(2);
1023 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1024 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1025 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1026 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1028 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1029 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1030 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1031 let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1033 // Queue up two payments - one will be delivered right away, one immediately goes into the
1034 // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1035 // immediately after a CS. By setting failing the monitor update failure from the CS (which
1036 // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1037 // generation during RAA while in monitor-update-failed state.
1038 nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1039 check_added_monitors!(nodes[0], 1);
1040 nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1041 check_added_monitors!(nodes[0], 0);
1043 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1044 assert_eq!(events.len(), 1);
1045 let payment_event = SendEvent::from_event(events.pop().unwrap());
1046 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1047 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1048 check_added_monitors!(nodes[1], 1);
1050 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1051 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1052 check_added_monitors!(nodes[0], 1);
1053 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1054 assert_eq!(events.len(), 1);
1055 let payment_event = SendEvent::from_event(events.pop().unwrap());
1057 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1058 check_added_monitors!(nodes[0], 1);
1059 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1061 // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1062 // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1063 // then restore channel monitor updates.
1064 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1065 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1066 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1067 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1068 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1069 check_added_monitors!(nodes[1], 1);
1071 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1072 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1073 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1074 check_added_monitors!(nodes[1], 1);
1076 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1077 nodes[1].node.test_restore_channel_monitor();
1078 // nodes[1] should be AwaitingRAA here!
1079 check_added_monitors!(nodes[1], 1);
1080 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1081 expect_pending_htlcs_forwardable!(nodes[1]);
1082 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1084 // We send a third payment here, which is somewhat of a redundant test, but the
1085 // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1086 // commitment transaction states) whereas here we can explicitly check for it.
1087 nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1088 check_added_monitors!(nodes[0], 0);
1089 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1091 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1092 check_added_monitors!(nodes[0], 1);
1093 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1094 assert_eq!(events.len(), 1);
1095 let payment_event = SendEvent::from_event(events.pop().unwrap());
1097 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1098 check_added_monitors!(nodes[0], 1);
1099 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1101 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1102 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1103 check_added_monitors!(nodes[1], 1);
1104 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1106 // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1107 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1108 check_added_monitors!(nodes[1], 1);
1109 expect_pending_htlcs_forwardable!(nodes[1]);
1110 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1111 let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1113 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1114 check_added_monitors!(nodes[0], 1);
1116 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1117 check_added_monitors!(nodes[0], 1);
1118 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1120 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1121 check_added_monitors!(nodes[1], 1);
1122 expect_pending_htlcs_forwardable!(nodes[1]);
1123 expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1125 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1126 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1127 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1131 fn claim_while_disconnected_monitor_update_fail() {
1132 // Test for claiming a payment while disconnected and then having the resulting
1133 // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1134 // contrived case for nodes with network instability.
1135 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1136 // code introduced a regression in this test (specifically, this caught a removal of the
1137 // channel_reestablish handling ensuring the order was sensical given the messages used).
1138 let chanmon_cfgs = create_chanmon_cfgs(2);
1139 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1140 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1141 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1142 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1144 // Forward a payment for B to claim
1145 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1147 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1148 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1150 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1151 check_added_monitors!(nodes[1], 1);
1153 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1154 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1156 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1157 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1159 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1160 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1162 // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1164 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1166 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1167 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1168 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1169 check_added_monitors!(nodes[1], 1);
1170 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1172 // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1173 // the monitor still failed
1174 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1175 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1176 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1177 check_added_monitors!(nodes[0], 1);
1179 let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1180 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1181 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1182 check_added_monitors!(nodes[1], 1);
1183 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1184 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1185 // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1186 // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1188 // Now un-fail the monitor, which will result in B sending its original commitment update,
1189 // receiving the commitment update from A, and the resulting commitment dances.
1190 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1191 nodes[1].node.test_restore_channel_monitor();
1192 check_added_monitors!(nodes[1], 1);
1194 let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1195 assert_eq!(bs_msgs.len(), 2);
1198 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1199 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1200 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1201 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1202 check_added_monitors!(nodes[0], 1);
1204 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1205 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1206 check_added_monitors!(nodes[1], 1);
1208 _ => panic!("Unexpected event"),
1212 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1213 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1214 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1215 check_added_monitors!(nodes[0], 1);
1217 _ => panic!("Unexpected event"),
1220 let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1222 let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1223 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1224 check_added_monitors!(nodes[0], 1);
1225 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1227 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1228 check_added_monitors!(nodes[1], 1);
1229 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1230 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1231 check_added_monitors!(nodes[1], 1);
1233 expect_pending_htlcs_forwardable!(nodes[1]);
1234 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1236 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1237 check_added_monitors!(nodes[0], 1);
1239 let events = nodes[0].node.get_and_clear_pending_events();
1240 assert_eq!(events.len(), 1);
1242 Event::PaymentSent { ref payment_preimage } => {
1243 assert_eq!(*payment_preimage, payment_preimage_1);
1245 _ => panic!("Unexpected event"),
1248 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1252 fn monitor_failed_no_reestablish_response() {
1253 // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1254 // response to a commitment_signed.
1255 // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1256 // debug_assert!() failure in channel_reestablish handling.
1257 let chanmon_cfgs = create_chanmon_cfgs(2);
1258 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1259 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1260 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1261 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1263 // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1265 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1266 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1267 nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1268 check_added_monitors!(nodes[0], 1);
1270 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1271 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1272 assert_eq!(events.len(), 1);
1273 let payment_event = SendEvent::from_event(events.pop().unwrap());
1274 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1275 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1276 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1277 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1278 check_added_monitors!(nodes[1], 1);
1280 // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1281 // is still failing to update monitors.
1282 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1283 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1285 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1286 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1288 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1289 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1291 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1292 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1294 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1295 nodes[1].node.test_restore_channel_monitor();
1296 check_added_monitors!(nodes[1], 1);
1297 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1299 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1300 check_added_monitors!(nodes[0], 1);
1301 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1302 check_added_monitors!(nodes[0], 1);
1304 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1305 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1306 check_added_monitors!(nodes[1], 1);
1308 expect_pending_htlcs_forwardable!(nodes[1]);
1309 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1311 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1315 fn first_message_on_recv_ordering() {
1316 // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1317 // messages, we're willing to flip the order of response messages if neccessary in resposne to
1318 // a commitment_signed which needs to send an RAA first.
1319 // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1320 // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1321 // response. To do this, we start routing two payments, with the final RAA for the first being
1322 // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1323 // have no pending response but will want to send a RAA/CS (with the updates for the second
1324 // payment applied).
1325 // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1326 let chanmon_cfgs = create_chanmon_cfgs(2);
1327 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1328 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1329 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1330 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1332 // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1333 // can deliver it and fail the monitor update.
1334 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1335 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1336 nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1337 check_added_monitors!(nodes[0], 1);
1339 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1340 assert_eq!(events.len(), 1);
1341 let payment_event = SendEvent::from_event(events.pop().unwrap());
1342 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1343 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1344 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1345 check_added_monitors!(nodes[1], 1);
1346 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1348 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1349 check_added_monitors!(nodes[0], 1);
1350 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1351 check_added_monitors!(nodes[0], 1);
1353 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1355 // Route the second payment, generating an update_add_htlc/commitment_signed
1356 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1357 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1358 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1359 check_added_monitors!(nodes[0], 1);
1360 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1361 assert_eq!(events.len(), 1);
1362 let payment_event = SendEvent::from_event(events.pop().unwrap());
1363 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1365 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1367 // Deliver the final RAA for the first payment, which does not require a response. RAAs
1368 // generally require a commitment_signed, so the fact that we're expecting an opposite response
1369 // to the next message also tests resetting the delivery order.
1370 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1371 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1372 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1373 check_added_monitors!(nodes[1], 1);
1375 // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1376 // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1377 // the appropriate HTLC acceptance).
1378 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1379 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1380 check_added_monitors!(nodes[1], 1);
1381 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1382 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1384 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1385 nodes[1].node.test_restore_channel_monitor();
1386 check_added_monitors!(nodes[1], 1);
1388 expect_pending_htlcs_forwardable!(nodes[1]);
1389 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1391 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1392 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1393 check_added_monitors!(nodes[0], 1);
1394 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1395 check_added_monitors!(nodes[0], 1);
1397 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1398 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1399 check_added_monitors!(nodes[1], 1);
1401 expect_pending_htlcs_forwardable!(nodes[1]);
1402 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1404 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1405 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1409 fn test_monitor_update_fail_claim() {
1410 // Basic test for monitor update failures when processing claim_funds calls.
1411 // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1412 // update to claim the payment. We then send a payment C->B->A, making the forward of this
1413 // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1414 // updating and claim the payment on B.
1415 let chanmon_cfgs = create_chanmon_cfgs(3);
1416 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1417 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1418 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1419 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1420 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1422 // Rebalance a bit so that we can send backwards from 3 to 2.
1423 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1425 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1427 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1428 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1429 check_added_monitors!(nodes[1], 1);
1431 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1432 let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1433 nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1434 check_added_monitors!(nodes[2], 1);
1436 // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1437 // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1438 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1440 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1441 assert_eq!(events.len(), 1);
1442 let payment_event = SendEvent::from_event(events.pop().unwrap());
1443 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1444 let events = nodes[1].node.get_and_clear_pending_msg_events();
1445 assert_eq!(events.len(), 0);
1446 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1447 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1449 let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1450 nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1451 commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1453 let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1454 assert_eq!(msg_events.len(), 1);
1455 match msg_events[0] {
1456 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1457 assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1458 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1460 _ => panic!("Unexpected event"),
1463 let events = nodes[2].node.get_and_clear_pending_events();
1464 assert_eq!(events.len(), 1);
1465 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1466 assert_eq!(payment_hash, payment_hash_2);
1467 assert!(!rejected_by_dest);
1468 } else { panic!("Unexpected event!"); }
1470 // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1471 nodes[1].node.test_restore_channel_monitor();
1472 check_added_monitors!(nodes[1], 1);
1474 let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1475 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1476 commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1478 let events = nodes[0].node.get_and_clear_pending_events();
1479 assert_eq!(events.len(), 1);
1480 if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1481 assert_eq!(payment_preimage, payment_preimage_1);
1482 } else { panic!("Unexpected event!"); }
1486 fn test_monitor_update_on_pending_forwards() {
1487 // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1488 // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1489 // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1490 // from C to A will be pending a forward to A.
1491 let chanmon_cfgs = create_chanmon_cfgs(3);
1492 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1493 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1494 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1495 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1496 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1498 // Rebalance a bit so that we can send backwards from 3 to 1.
1499 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1501 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1502 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1503 expect_pending_htlcs_forwardable!(nodes[2]);
1504 check_added_monitors!(nodes[2], 1);
1506 let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1507 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1508 commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1509 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1511 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1512 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1513 nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1514 check_added_monitors!(nodes[2], 1);
1516 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1517 assert_eq!(events.len(), 1);
1518 let payment_event = SendEvent::from_event(events.pop().unwrap());
1519 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1520 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1522 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1523 expect_pending_htlcs_forwardable!(nodes[1]);
1524 check_added_monitors!(nodes[1], 1);
1525 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1526 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1528 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1529 nodes[1].node.test_restore_channel_monitor();
1530 check_added_monitors!(nodes[1], 1);
1532 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1533 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1534 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1535 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1537 let events = nodes[0].node.get_and_clear_pending_events();
1538 assert_eq!(events.len(), 2);
1539 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1540 assert_eq!(payment_hash, payment_hash_1);
1541 assert!(rejected_by_dest);
1542 } else { panic!("Unexpected event!"); }
1544 Event::PendingHTLCsForwardable { .. } => { },
1545 _ => panic!("Unexpected event"),
1547 nodes[0].node.process_pending_htlc_forwards();
1548 expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1550 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1554 fn monitor_update_claim_fail_no_response() {
1555 // Test for claim_funds resulting in both a monitor update failure and no message response (due
1556 // to channel being AwaitingRAA).
1557 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1559 let chanmon_cfgs = create_chanmon_cfgs(2);
1560 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1561 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1562 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1563 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1565 // Forward a payment for B to claim
1566 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1568 // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1569 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1570 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1571 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1572 check_added_monitors!(nodes[0], 1);
1574 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1575 assert_eq!(events.len(), 1);
1576 let payment_event = SendEvent::from_event(events.pop().unwrap());
1577 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1578 let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1580 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1581 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1582 check_added_monitors!(nodes[1], 1);
1583 let events = nodes[1].node.get_and_clear_pending_msg_events();
1584 assert_eq!(events.len(), 0);
1585 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1587 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1588 nodes[1].node.test_restore_channel_monitor();
1589 check_added_monitors!(nodes[1], 1);
1590 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1592 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1593 check_added_monitors!(nodes[1], 1);
1594 expect_pending_htlcs_forwardable!(nodes[1]);
1595 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1597 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1598 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1599 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1601 let events = nodes[0].node.get_and_clear_pending_events();
1602 assert_eq!(events.len(), 1);
1604 Event::PaymentSent { ref payment_preimage } => {
1605 assert_eq!(*payment_preimage, payment_preimage_1);
1607 _ => panic!("Unexpected event"),
1610 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1613 // Note that restore_between_fails with !fail_on_generate is useless
1614 // Also note that !fail_on_generate && !fail_on_signed is useless
1615 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1616 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1617 // restore_b_before_conf has no meaning if !confirm_a_first
1618 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1619 // Test that if the monitor update generated by funding_transaction_generated fails we continue
1620 // the channel setup happily after the update is restored.
1621 let chanmon_cfgs = create_chanmon_cfgs(2);
1622 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1623 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1624 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1626 nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1627 nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1628 nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1630 let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1632 if fail_on_generate {
1633 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1635 nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1636 check_added_monitors!(nodes[0], 1);
1638 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1639 nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1640 check_added_monitors!(nodes[1], 1);
1642 if restore_between_fails {
1643 assert!(fail_on_generate);
1644 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1645 nodes[0].node.test_restore_channel_monitor();
1646 check_added_monitors!(nodes[0], 1);
1647 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1648 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1652 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1654 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1655 assert!(fail_on_generate); // Somebody has to fail
1657 nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1658 if fail_on_signed || !restore_between_fails {
1659 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1660 if fail_on_generate && !restore_between_fails {
1661 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1662 check_added_monitors!(nodes[0], 1);
1664 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1665 check_added_monitors!(nodes[0], 1);
1667 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1668 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1669 nodes[0].node.test_restore_channel_monitor();
1672 check_added_monitors!(nodes[0], 1);
1674 let events = nodes[0].node.get_and_clear_pending_events();
1675 assert_eq!(events.len(), 1);
1677 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1678 assert_eq!(user_channel_id, 43);
1679 assert_eq!(*funding_txo, funding_output);
1681 _ => panic!("Unexpected event"),
1684 if confirm_a_first {
1685 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1686 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1688 assert!(!restore_b_before_conf);
1689 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1690 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1693 // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1694 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1695 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1696 reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1697 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1698 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1700 if !restore_b_before_conf {
1701 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1702 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1703 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1706 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1707 nodes[1].node.test_restore_channel_monitor();
1708 check_added_monitors!(nodes[1], 1);
1710 let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1711 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1713 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1714 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1715 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1717 if restore_b_before_conf {
1718 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1720 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1721 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1723 for node in nodes.iter() {
1724 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1725 node.router.handle_channel_update(&as_update).unwrap();
1726 node.router.handle_channel_update(&bs_update).unwrap();
1729 send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1730 close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1734 fn during_funding_monitor_fail() {
1735 do_during_funding_monitor_fail(false, false, true, true, true);
1736 do_during_funding_monitor_fail(true, false, true, false, false);
1737 do_during_funding_monitor_fail(true, true, true, true, false);
1738 do_during_funding_monitor_fail(true, true, false, false, false);