1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
4 //! here. See also the chanmon_fail_consistency fuzz test.
6 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash};
7 use ln::channelmonitor::ChannelMonitorUpdateErr;
8 use ln::features::InitFeatures;
10 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
11 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
12 use util::errors::APIError;
14 use bitcoin_hashes::sha256::Hash as Sha256;
15 use bitcoin_hashes::Hash;
17 use ln::functional_test_utils::*;
20 fn test_simple_monitor_permanent_update_fail() {
21 // Test that we handle a simple permanent monitor update failure
22 let chanmon_cfgs = create_chanmon_cfgs(2);
23 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
24 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
25 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
26 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
28 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
29 let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
31 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
32 if let Err(APIError::ChannelUnavailable {..}) = nodes[0].node.send_payment(route, payment_hash_1) {} else { panic!(); }
33 check_added_monitors!(nodes[0], 1);
35 let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
36 assert_eq!(events_1.len(), 2);
38 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
39 _ => panic!("Unexpected event"),
42 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
43 _ => panic!("Unexpected event"),
46 // TODO: Once we hit the chain with the failure transaction we should check that we get a
47 // PaymentFailed event
49 assert_eq!(nodes[0].node.list_channels().len(), 0);
52 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
53 // Test that we can recover from a simple temporary monitor update failure optionally with
54 // a disconnect in between
55 let chanmon_cfgs = create_chanmon_cfgs(2);
56 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
57 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
58 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
59 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
61 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
62 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
64 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
65 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_1) {} else { panic!(); }
66 check_added_monitors!(nodes[0], 1);
68 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
69 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
70 assert_eq!(nodes[0].node.list_channels().len(), 1);
73 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
74 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
75 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
78 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
79 nodes[0].node.test_restore_channel_monitor();
80 check_added_monitors!(nodes[0], 1);
82 let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
83 assert_eq!(events_2.len(), 1);
84 let payment_event = SendEvent::from_event(events_2.pop().unwrap());
85 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
86 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
87 commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
89 expect_pending_htlcs_forwardable!(nodes[1]);
91 let events_3 = nodes[1].node.get_and_clear_pending_events();
92 assert_eq!(events_3.len(), 1);
94 Event::PaymentReceived { ref payment_hash, amt } => {
95 assert_eq!(payment_hash_1, *payment_hash);
96 assert_eq!(amt, 1000000);
98 _ => panic!("Unexpected event"),
101 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
103 // Now set it to failed again...
104 let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
105 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
106 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route, payment_hash_2) {} else { panic!(); }
107 check_added_monitors!(nodes[0], 1);
109 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
110 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
111 assert_eq!(nodes[0].node.list_channels().len(), 1);
114 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
115 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
116 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
119 // ...and make sure we can force-close a TemporaryFailure channel with a PermanentFailure
120 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
121 nodes[0].node.test_restore_channel_monitor();
122 check_added_monitors!(nodes[0], 1);
123 check_closed_broadcast!(nodes[0], false);
125 // TODO: Once we hit the chain with the failure transaction we should check that we get a
126 // PaymentFailed event
128 assert_eq!(nodes[0].node.list_channels().len(), 0);
132 fn test_simple_monitor_temporary_update_fail() {
133 do_test_simple_monitor_temporary_update_fail(false);
134 do_test_simple_monitor_temporary_update_fail(true);
137 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
138 let disconnect_flags = 8 | 16;
140 // Test that we can recover from a temporary monitor update failure with some in-flight
141 // HTLCs going on at the same time potentially with some disconnection thrown in.
142 // * First we route a payment, then get a temporary monitor update failure when trying to
143 // route a second payment. We then claim the first payment.
144 // * If disconnect_count is set, we will disconnect at this point (which is likely as
145 // TemporaryFailure likely indicates net disconnect which resulted in failing to update
146 // the ChannelMonitor on a watchtower).
147 // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
148 // immediately, otherwise we wait disconnect and deliver them via the reconnect
149 // channel_reestablish processing (ie disconnect_count & 16 makes no sense if
150 // disconnect_count & !disconnect_flags is 0).
151 // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
152 // through message sending, potentially disconnect/reconnecting multiple times based on
153 // disconnect_count, to get the update_fulfill_htlc through.
154 // * We then walk through more message exchanges to get the original update_add_htlc
155 // through, swapping message ordering based on disconnect_count & 8 and optionally
156 // disconnect/reconnecting based on disconnect_count.
157 let chanmon_cfgs = create_chanmon_cfgs(2);
158 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
159 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
160 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
161 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
163 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
165 // Now try to send a second payment which will fail to send
166 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
167 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
169 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
170 if let Err(APIError::MonitorUpdateFailed) = nodes[0].node.send_payment(route.clone(), payment_hash_2) {} else { panic!(); }
171 check_added_monitors!(nodes[0], 1);
173 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
174 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
175 assert_eq!(nodes[0].node.list_channels().len(), 1);
177 // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
178 // but nodes[0] won't respond since it is frozen.
179 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
180 check_added_monitors!(nodes[1], 1);
181 let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
182 assert_eq!(events_2.len(), 1);
183 let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
184 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
185 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
186 assert!(update_add_htlcs.is_empty());
187 assert_eq!(update_fulfill_htlcs.len(), 1);
188 assert!(update_fail_htlcs.is_empty());
189 assert!(update_fail_malformed_htlcs.is_empty());
190 assert!(update_fee.is_none());
192 if (disconnect_count & 16) == 0 {
193 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
194 let events_3 = nodes[0].node.get_and_clear_pending_events();
195 assert_eq!(events_3.len(), 1);
197 Event::PaymentSent { ref payment_preimage } => {
198 assert_eq!(*payment_preimage, payment_preimage_1);
200 _ => panic!("Unexpected event"),
203 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
204 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
205 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
208 (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
210 _ => panic!("Unexpected event"),
213 if disconnect_count & !disconnect_flags > 0 {
214 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
215 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
218 // Now fix monitor updating...
219 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
220 nodes[0].node.test_restore_channel_monitor();
221 check_added_monitors!(nodes[0], 1);
223 macro_rules! disconnect_reconnect_peers { () => { {
224 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
225 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
227 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
228 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
229 assert_eq!(reestablish_1.len(), 1);
230 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
231 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
232 assert_eq!(reestablish_2.len(), 1);
234 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
235 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
236 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
237 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
239 assert!(as_resp.0.is_none());
240 assert!(bs_resp.0.is_none());
242 (reestablish_1, reestablish_2, as_resp, bs_resp)
245 let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
246 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
247 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
249 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
250 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
251 assert_eq!(reestablish_1.len(), 1);
252 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
253 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
254 assert_eq!(reestablish_2.len(), 1);
256 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
257 check_added_monitors!(nodes[0], 0);
258 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
259 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
260 check_added_monitors!(nodes[1], 0);
261 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
263 assert!(as_resp.0.is_none());
264 assert!(bs_resp.0.is_none());
266 assert!(bs_resp.1.is_none());
267 if (disconnect_count & 16) == 0 {
268 assert!(bs_resp.2.is_none());
270 assert!(as_resp.1.is_some());
271 assert!(as_resp.2.is_some());
272 assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
274 assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
275 assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
276 assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
277 assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
278 assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
279 assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
281 assert!(as_resp.1.is_none());
283 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
284 let events_3 = nodes[0].node.get_and_clear_pending_events();
285 assert_eq!(events_3.len(), 1);
287 Event::PaymentSent { ref payment_preimage } => {
288 assert_eq!(*payment_preimage, payment_preimage_1);
290 _ => panic!("Unexpected event"),
293 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
294 let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
295 // No commitment_signed so get_event_msg's assert(len == 1) passes
296 check_added_monitors!(nodes[0], 1);
298 as_resp.1 = Some(as_resp_raa);
302 if disconnect_count & !disconnect_flags > 1 {
303 let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
305 if (disconnect_count & 16) == 0 {
306 assert!(reestablish_1 == second_reestablish_1);
307 assert!(reestablish_2 == second_reestablish_2);
309 assert!(as_resp == second_as_resp);
310 assert!(bs_resp == second_bs_resp);
313 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
315 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
316 assert_eq!(events_4.len(), 2);
317 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
318 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
319 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
322 _ => panic!("Unexpected event"),
326 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
328 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
329 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
330 let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
331 // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
332 check_added_monitors!(nodes[1], 1);
334 if disconnect_count & !disconnect_flags > 2 {
335 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
337 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
338 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
340 assert!(as_resp.2.is_none());
341 assert!(bs_resp.2.is_none());
344 let as_commitment_update;
345 let bs_second_commitment_update;
347 macro_rules! handle_bs_raa { () => {
348 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
349 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
350 assert!(as_commitment_update.update_add_htlcs.is_empty());
351 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
352 assert!(as_commitment_update.update_fail_htlcs.is_empty());
353 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
354 assert!(as_commitment_update.update_fee.is_none());
355 check_added_monitors!(nodes[0], 1);
358 macro_rules! handle_initial_raa { () => {
359 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
360 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
361 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
362 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
363 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
364 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
365 assert!(bs_second_commitment_update.update_fee.is_none());
366 check_added_monitors!(nodes[1], 1);
369 if (disconnect_count & 8) == 0 {
372 if disconnect_count & !disconnect_flags > 3 {
373 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
375 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
376 assert!(bs_resp.1.is_none());
378 assert!(as_resp.2.unwrap() == as_commitment_update);
379 assert!(bs_resp.2.is_none());
381 assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
384 handle_initial_raa!();
386 if disconnect_count & !disconnect_flags > 4 {
387 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
389 assert!(as_resp.1.is_none());
390 assert!(bs_resp.1.is_none());
392 assert!(as_resp.2.unwrap() == as_commitment_update);
393 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
396 handle_initial_raa!();
398 if disconnect_count & !disconnect_flags > 3 {
399 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
401 assert!(as_resp.1.is_none());
402 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
404 assert!(as_resp.2.is_none());
405 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
407 assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
412 if disconnect_count & !disconnect_flags > 4 {
413 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
415 assert!(as_resp.1.is_none());
416 assert!(bs_resp.1.is_none());
418 assert!(as_resp.2.unwrap() == as_commitment_update);
419 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
423 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
424 let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
425 // No commitment_signed so get_event_msg's assert(len == 1) passes
426 check_added_monitors!(nodes[0], 1);
428 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
429 let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
430 // No commitment_signed so get_event_msg's assert(len == 1) passes
431 check_added_monitors!(nodes[1], 1);
433 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
434 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
435 check_added_monitors!(nodes[1], 1);
437 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
438 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
439 check_added_monitors!(nodes[0], 1);
441 expect_pending_htlcs_forwardable!(nodes[1]);
443 let events_5 = nodes[1].node.get_and_clear_pending_events();
444 assert_eq!(events_5.len(), 1);
446 Event::PaymentReceived { ref payment_hash, amt } => {
447 assert_eq!(payment_hash_2, *payment_hash);
448 assert_eq!(amt, 1000000);
450 _ => panic!("Unexpected event"),
453 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
457 fn test_monitor_temporary_update_fail_a() {
458 do_test_monitor_temporary_update_fail(0);
459 do_test_monitor_temporary_update_fail(1);
460 do_test_monitor_temporary_update_fail(2);
461 do_test_monitor_temporary_update_fail(3);
462 do_test_monitor_temporary_update_fail(4);
463 do_test_monitor_temporary_update_fail(5);
467 fn test_monitor_temporary_update_fail_b() {
468 do_test_monitor_temporary_update_fail(2 | 8);
469 do_test_monitor_temporary_update_fail(3 | 8);
470 do_test_monitor_temporary_update_fail(4 | 8);
471 do_test_monitor_temporary_update_fail(5 | 8);
475 fn test_monitor_temporary_update_fail_c() {
476 do_test_monitor_temporary_update_fail(1 | 16);
477 do_test_monitor_temporary_update_fail(2 | 16);
478 do_test_monitor_temporary_update_fail(3 | 16);
479 do_test_monitor_temporary_update_fail(2 | 8 | 16);
480 do_test_monitor_temporary_update_fail(3 | 8 | 16);
484 fn test_monitor_update_fail_cs() {
485 // Tests handling of a monitor update failure when processing an incoming commitment_signed
486 let chanmon_cfgs = create_chanmon_cfgs(2);
487 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
488 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
489 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
490 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
492 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
493 let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
494 nodes[0].node.send_payment(route, our_payment_hash).unwrap();
495 check_added_monitors!(nodes[0], 1);
497 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
498 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
500 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
501 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
502 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
503 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
504 check_added_monitors!(nodes[1], 1);
505 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
507 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
508 nodes[1].node.test_restore_channel_monitor();
509 check_added_monitors!(nodes[1], 1);
510 let responses = nodes[1].node.get_and_clear_pending_msg_events();
511 assert_eq!(responses.len(), 2);
514 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
515 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
516 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
517 check_added_monitors!(nodes[0], 1);
519 _ => panic!("Unexpected event"),
522 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
523 assert!(updates.update_add_htlcs.is_empty());
524 assert!(updates.update_fulfill_htlcs.is_empty());
525 assert!(updates.update_fail_htlcs.is_empty());
526 assert!(updates.update_fail_malformed_htlcs.is_empty());
527 assert!(updates.update_fee.is_none());
528 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
530 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
531 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
532 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
533 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
534 check_added_monitors!(nodes[0], 1);
535 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
537 _ => panic!("Unexpected event"),
540 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
541 nodes[0].node.test_restore_channel_monitor();
542 check_added_monitors!(nodes[0], 1);
544 let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
545 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
546 check_added_monitors!(nodes[1], 1);
548 expect_pending_htlcs_forwardable!(nodes[1]);
550 let events = nodes[1].node.get_and_clear_pending_events();
551 assert_eq!(events.len(), 1);
553 Event::PaymentReceived { payment_hash, amt } => {
554 assert_eq!(payment_hash, our_payment_hash);
555 assert_eq!(amt, 1000000);
557 _ => panic!("Unexpected event"),
560 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
564 fn test_monitor_update_fail_no_rebroadcast() {
565 // Tests handling of a monitor update failure when no message rebroadcasting on
566 // test_restore_channel_monitor() is required. Backported from
567 // chanmon_fail_consistency fuzz tests.
568 let chanmon_cfgs = create_chanmon_cfgs(2);
569 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
570 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
571 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
572 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
574 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
575 let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
576 nodes[0].node.send_payment(route, our_payment_hash).unwrap();
577 check_added_monitors!(nodes[0], 1);
579 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
580 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
581 let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
583 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
584 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
585 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
586 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
587 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
588 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
589 check_added_monitors!(nodes[1], 1);
591 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
592 nodes[1].node.test_restore_channel_monitor();
593 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
594 check_added_monitors!(nodes[1], 1);
595 expect_pending_htlcs_forwardable!(nodes[1]);
597 let events = nodes[1].node.get_and_clear_pending_events();
598 assert_eq!(events.len(), 1);
600 Event::PaymentReceived { payment_hash, .. } => {
601 assert_eq!(payment_hash, our_payment_hash);
603 _ => panic!("Unexpected event"),
606 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
610 fn test_monitor_update_raa_while_paused() {
611 // Tests handling of an RAA while monitor updating has already been marked failed.
612 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
613 let chanmon_cfgs = create_chanmon_cfgs(2);
614 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
615 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
616 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
617 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
619 send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
621 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
622 let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
623 nodes[0].node.send_payment(route, our_payment_hash_1).unwrap();
624 check_added_monitors!(nodes[0], 1);
625 let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
627 let route = nodes[1].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
628 let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
629 nodes[1].node.send_payment(route, our_payment_hash_2).unwrap();
630 check_added_monitors!(nodes[1], 1);
631 let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
633 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
634 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
635 check_added_monitors!(nodes[1], 1);
636 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
638 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
639 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
640 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
641 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
642 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
643 check_added_monitors!(nodes[0], 1);
645 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
646 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
647 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
648 check_added_monitors!(nodes[0], 1);
650 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
651 nodes[0].node.test_restore_channel_monitor();
652 check_added_monitors!(nodes[0], 1);
654 let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
655 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
656 check_added_monitors!(nodes[1], 1);
657 let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
659 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
660 check_added_monitors!(nodes[1], 1);
661 let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
663 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
664 check_added_monitors!(nodes[0], 1);
665 let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
667 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
668 check_added_monitors!(nodes[0], 1);
669 expect_pending_htlcs_forwardable!(nodes[0]);
670 expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
672 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
673 check_added_monitors!(nodes[1], 1);
674 expect_pending_htlcs_forwardable!(nodes[1]);
675 expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
677 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
678 claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
681 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
682 // Tests handling of a monitor update failure when processing an incoming RAA
683 let chanmon_cfgs = create_chanmon_cfgs(3);
684 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
685 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
686 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
687 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
688 let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
690 // Rebalance a bit so that we can send backwards from 2 to 1.
691 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
693 // Route a first payment that we'll fail backwards
694 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
696 // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
697 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
698 expect_pending_htlcs_forwardable!(nodes[2]);
699 check_added_monitors!(nodes[2], 1);
701 let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
702 assert!(updates.update_add_htlcs.is_empty());
703 assert!(updates.update_fulfill_htlcs.is_empty());
704 assert_eq!(updates.update_fail_htlcs.len(), 1);
705 assert!(updates.update_fail_malformed_htlcs.is_empty());
706 assert!(updates.update_fee.is_none());
707 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
709 let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
710 check_added_monitors!(nodes[0], 0);
712 // While the second channel is AwaitingRAA, forward a second payment to get it into the
714 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
715 let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
716 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
717 check_added_monitors!(nodes[0], 1);
719 let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
720 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
721 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
723 expect_pending_htlcs_forwardable!(nodes[1]);
724 check_added_monitors!(nodes[1], 0);
725 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
727 // Now fail monitor updating.
728 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
729 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
730 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
731 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
732 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
733 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
734 check_added_monitors!(nodes[1], 1);
736 // Attempt to forward a third payment but fail due to the second channel being unavailable
739 let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
740 let route = nodes[0].router.get_route(&nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
741 nodes[0].node.send_payment(route, payment_hash_3).unwrap();
742 check_added_monitors!(nodes[0], 1);
744 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
745 send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
746 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
747 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
748 check_added_monitors!(nodes[1], 0);
750 let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
751 assert_eq!(events_2.len(), 1);
752 match events_2.remove(0) {
753 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
754 assert_eq!(node_id, nodes[0].node.get_our_node_id());
755 assert!(updates.update_fulfill_htlcs.is_empty());
756 assert_eq!(updates.update_fail_htlcs.len(), 1);
757 assert!(updates.update_fail_malformed_htlcs.is_empty());
758 assert!(updates.update_add_htlcs.is_empty());
759 assert!(updates.update_fee.is_none());
761 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
762 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
764 let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
765 assert_eq!(msg_events.len(), 1);
766 match msg_events[0] {
767 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
768 assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
769 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
771 _ => panic!("Unexpected event"),
774 let events = nodes[0].node.get_and_clear_pending_events();
775 assert_eq!(events.len(), 1);
776 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
777 assert_eq!(payment_hash, payment_hash_3);
778 assert!(!rejected_by_dest);
779 } else { panic!("Unexpected event!"); }
781 _ => panic!("Unexpected event type!"),
784 let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
785 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
786 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
787 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
788 nodes[2].node.send_payment(route, payment_hash_4).unwrap();
789 check_added_monitors!(nodes[2], 1);
791 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
792 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
793 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
794 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
795 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
796 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
797 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
798 (Some(payment_preimage_4), Some(payment_hash_4))
799 } else { (None, None) };
801 // Restore monitor updating, ensuring we immediately get a fail-back update and a
802 // update_add update.
803 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
804 nodes[1].node.test_restore_channel_monitor();
805 check_added_monitors!(nodes[1], 1);
806 expect_pending_htlcs_forwardable!(nodes[1]);
807 check_added_monitors!(nodes[1], 1);
809 let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
810 if test_ignore_second_cs {
811 assert_eq!(events_3.len(), 3);
813 assert_eq!(events_3.len(), 2);
816 // Note that the ordering of the events for different nodes is non-prescriptive, though the
817 // ordering of the two events that both go to nodes[2] have to stay in the same order.
818 let messages_a = match events_3.pop().unwrap() {
819 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
820 assert_eq!(node_id, nodes[0].node.get_our_node_id());
821 assert!(updates.update_fulfill_htlcs.is_empty());
822 assert_eq!(updates.update_fail_htlcs.len(), 1);
823 assert!(updates.update_fail_malformed_htlcs.is_empty());
824 assert!(updates.update_add_htlcs.is_empty());
825 assert!(updates.update_fee.is_none());
826 (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
828 _ => panic!("Unexpected event type!"),
830 let raa = if test_ignore_second_cs {
831 match events_3.remove(1) {
832 MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
833 assert_eq!(node_id, nodes[2].node.get_our_node_id());
836 _ => panic!("Unexpected event"),
839 let send_event_b = SendEvent::from_event(events_3.remove(0));
840 assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
842 // Now deliver the new messages...
844 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
845 commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
846 let events_4 = nodes[0].node.get_and_clear_pending_events();
847 assert_eq!(events_4.len(), 1);
848 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
849 assert_eq!(payment_hash, payment_hash_1);
850 assert!(rejected_by_dest);
851 } else { panic!("Unexpected event!"); }
853 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
854 if test_ignore_second_cs {
855 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
856 check_added_monitors!(nodes[2], 1);
857 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
858 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
859 check_added_monitors!(nodes[2], 1);
860 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
861 assert!(bs_cs.update_add_htlcs.is_empty());
862 assert!(bs_cs.update_fail_htlcs.is_empty());
863 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
864 assert!(bs_cs.update_fulfill_htlcs.is_empty());
865 assert!(bs_cs.update_fee.is_none());
867 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
868 check_added_monitors!(nodes[1], 1);
869 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
870 assert!(as_cs.update_add_htlcs.is_empty());
871 assert!(as_cs.update_fail_htlcs.is_empty());
872 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
873 assert!(as_cs.update_fulfill_htlcs.is_empty());
874 assert!(as_cs.update_fee.is_none());
876 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
877 check_added_monitors!(nodes[1], 1);
878 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
880 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
881 check_added_monitors!(nodes[2], 1);
882 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
884 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
885 check_added_monitors!(nodes[2], 1);
886 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
888 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
889 check_added_monitors!(nodes[1], 1);
890 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
892 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
895 expect_pending_htlcs_forwardable!(nodes[2]);
897 let events_6 = nodes[2].node.get_and_clear_pending_events();
898 assert_eq!(events_6.len(), 1);
900 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
901 _ => panic!("Unexpected event"),
904 if test_ignore_second_cs {
905 expect_pending_htlcs_forwardable!(nodes[1]);
906 check_added_monitors!(nodes[1], 1);
908 send_event = SendEvent::from_node(&nodes[1]);
909 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
910 assert_eq!(send_event.msgs.len(), 1);
911 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
912 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
914 expect_pending_htlcs_forwardable!(nodes[0]);
916 let events_9 = nodes[0].node.get_and_clear_pending_events();
917 assert_eq!(events_9.len(), 1);
919 Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
920 _ => panic!("Unexpected event"),
922 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
925 claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
929 fn test_monitor_update_fail_raa() {
930 do_test_monitor_update_fail_raa(false);
931 do_test_monitor_update_fail_raa(true);
935 fn test_monitor_update_fail_reestablish() {
936 // Simple test for message retransmission after monitor update failure on
937 // channel_reestablish generating a monitor update (which comes from freeing holding cell
939 let chanmon_cfgs = create_chanmon_cfgs(3);
940 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
941 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
942 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
943 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
944 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
946 let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
948 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
949 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
951 assert!(nodes[2].node.claim_funds(our_payment_preimage, 1_000_000));
952 check_added_monitors!(nodes[2], 1);
953 let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
954 assert!(updates.update_add_htlcs.is_empty());
955 assert!(updates.update_fail_htlcs.is_empty());
956 assert!(updates.update_fail_malformed_htlcs.is_empty());
957 assert!(updates.update_fee.is_none());
958 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
959 nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
960 check_added_monitors!(nodes[1], 1);
961 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
962 commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
964 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
965 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
966 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
968 let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
969 let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
971 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
973 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
974 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
975 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
976 check_added_monitors!(nodes[1], 1);
978 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
979 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
981 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
982 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
984 assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
985 assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
987 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
989 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
990 check_added_monitors!(nodes[1], 0);
991 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
993 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
994 nodes[1].node.test_restore_channel_monitor();
995 check_added_monitors!(nodes[1], 1);
997 updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
998 assert!(updates.update_add_htlcs.is_empty());
999 assert!(updates.update_fail_htlcs.is_empty());
1000 assert!(updates.update_fail_malformed_htlcs.is_empty());
1001 assert!(updates.update_fee.is_none());
1002 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1003 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1004 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1006 let events = nodes[0].node.get_and_clear_pending_events();
1007 assert_eq!(events.len(), 1);
1009 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1010 _ => panic!("Unexpected event"),
1015 fn raa_no_response_awaiting_raa_state() {
1016 // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1017 // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1018 // in question (assuming it intends to respond with a CS after monitor updating is restored).
1019 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1020 let chanmon_cfgs = create_chanmon_cfgs(2);
1021 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1022 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1023 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1024 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1026 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1027 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1028 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1029 let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1031 // Queue up two payments - one will be delivered right away, one immediately goes into the
1032 // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1033 // immediately after a CS. By setting failing the monitor update failure from the CS (which
1034 // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1035 // generation during RAA while in monitor-update-failed state.
1036 nodes[0].node.send_payment(route.clone(), payment_hash_1).unwrap();
1037 check_added_monitors!(nodes[0], 1);
1038 nodes[0].node.send_payment(route.clone(), payment_hash_2).unwrap();
1039 check_added_monitors!(nodes[0], 0);
1041 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1042 assert_eq!(events.len(), 1);
1043 let payment_event = SendEvent::from_event(events.pop().unwrap());
1044 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1045 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1046 check_added_monitors!(nodes[1], 1);
1048 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1049 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1050 check_added_monitors!(nodes[0], 1);
1051 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1052 assert_eq!(events.len(), 1);
1053 let payment_event = SendEvent::from_event(events.pop().unwrap());
1055 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1056 check_added_monitors!(nodes[0], 1);
1057 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1059 // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1060 // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1061 // then restore channel monitor updates.
1062 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1063 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1064 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1065 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1066 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1067 check_added_monitors!(nodes[1], 1);
1069 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1070 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1071 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1072 check_added_monitors!(nodes[1], 1);
1074 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1075 nodes[1].node.test_restore_channel_monitor();
1076 // nodes[1] should be AwaitingRAA here!
1077 check_added_monitors!(nodes[1], 1);
1078 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1079 expect_pending_htlcs_forwardable!(nodes[1]);
1080 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1082 // We send a third payment here, which is somewhat of a redundant test, but the
1083 // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1084 // commitment transaction states) whereas here we can explicitly check for it.
1085 nodes[0].node.send_payment(route.clone(), payment_hash_3).unwrap();
1086 check_added_monitors!(nodes[0], 0);
1087 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1089 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1090 check_added_monitors!(nodes[0], 1);
1091 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1092 assert_eq!(events.len(), 1);
1093 let payment_event = SendEvent::from_event(events.pop().unwrap());
1095 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1096 check_added_monitors!(nodes[0], 1);
1097 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1099 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1100 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1101 check_added_monitors!(nodes[1], 1);
1102 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1104 // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1105 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1106 check_added_monitors!(nodes[1], 1);
1107 expect_pending_htlcs_forwardable!(nodes[1]);
1108 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1109 let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1111 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1112 check_added_monitors!(nodes[0], 1);
1114 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1115 check_added_monitors!(nodes[0], 1);
1116 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1118 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1119 check_added_monitors!(nodes[1], 1);
1120 expect_pending_htlcs_forwardable!(nodes[1]);
1121 expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1123 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1124 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1125 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1129 fn claim_while_disconnected_monitor_update_fail() {
1130 // Test for claiming a payment while disconnected and then having the resulting
1131 // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1132 // contrived case for nodes with network instability.
1133 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1134 // code introduced a regression in this test (specifically, this caught a removal of the
1135 // channel_reestablish handling ensuring the order was sensical given the messages used).
1136 let chanmon_cfgs = create_chanmon_cfgs(2);
1137 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1138 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1139 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1140 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1142 // Forward a payment for B to claim
1143 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1145 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1146 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1148 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1149 check_added_monitors!(nodes[1], 1);
1151 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1152 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1154 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1155 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1157 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1158 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1160 // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1162 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1164 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1165 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1166 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1167 check_added_monitors!(nodes[1], 1);
1168 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1170 // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1171 // the monitor still failed
1172 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1173 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1174 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1175 check_added_monitors!(nodes[0], 1);
1177 let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1178 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1179 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1180 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1181 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1182 // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1183 // until we've test_restore_channel_monitor'd and updated for the new commitment transaction.
1185 // Now un-fail the monitor, which will result in B sending its original commitment update,
1186 // receiving the commitment update from A, and the resulting commitment dances.
1187 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1188 nodes[1].node.test_restore_channel_monitor();
1189 check_added_monitors!(nodes[1], 1);
1191 let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1192 assert_eq!(bs_msgs.len(), 2);
1195 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1196 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1197 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1198 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1199 check_added_monitors!(nodes[0], 1);
1201 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1202 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1203 check_added_monitors!(nodes[1], 1);
1205 _ => panic!("Unexpected event"),
1209 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1210 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1211 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1212 check_added_monitors!(nodes[0], 1);
1214 _ => panic!("Unexpected event"),
1217 let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1219 let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1220 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1221 check_added_monitors!(nodes[0], 1);
1222 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1224 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1225 check_added_monitors!(nodes[1], 1);
1226 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1227 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1228 check_added_monitors!(nodes[1], 1);
1230 expect_pending_htlcs_forwardable!(nodes[1]);
1231 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1233 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1234 check_added_monitors!(nodes[0], 1);
1236 let events = nodes[0].node.get_and_clear_pending_events();
1237 assert_eq!(events.len(), 1);
1239 Event::PaymentSent { ref payment_preimage } => {
1240 assert_eq!(*payment_preimage, payment_preimage_1);
1242 _ => panic!("Unexpected event"),
1245 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1249 fn monitor_failed_no_reestablish_response() {
1250 // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1251 // response to a commitment_signed.
1252 // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1253 // debug_assert!() failure in channel_reestablish handling.
1254 let chanmon_cfgs = create_chanmon_cfgs(2);
1255 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1256 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1257 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1258 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1260 // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1262 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1263 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1264 nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1265 check_added_monitors!(nodes[0], 1);
1267 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1268 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1269 assert_eq!(events.len(), 1);
1270 let payment_event = SendEvent::from_event(events.pop().unwrap());
1271 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1272 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1273 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1274 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1275 check_added_monitors!(nodes[1], 1);
1277 // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1278 // is still failing to update monitors.
1279 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1280 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1282 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1283 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1285 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1286 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1288 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1289 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1291 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1292 nodes[1].node.test_restore_channel_monitor();
1293 check_added_monitors!(nodes[1], 1);
1294 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1296 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1297 check_added_monitors!(nodes[0], 1);
1298 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1299 check_added_monitors!(nodes[0], 1);
1301 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1302 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1303 check_added_monitors!(nodes[1], 1);
1305 expect_pending_htlcs_forwardable!(nodes[1]);
1306 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1308 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1312 fn first_message_on_recv_ordering() {
1313 // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1314 // messages, we're willing to flip the order of response messages if neccessary in resposne to
1315 // a commitment_signed which needs to send an RAA first.
1316 // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1317 // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1318 // response. To do this, we start routing two payments, with the final RAA for the first being
1319 // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1320 // have no pending response but will want to send a RAA/CS (with the updates for the second
1321 // payment applied).
1322 // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1323 let chanmon_cfgs = create_chanmon_cfgs(2);
1324 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1325 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1326 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1327 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1329 // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1330 // can deliver it and fail the monitor update.
1331 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1332 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1333 nodes[0].node.send_payment(route, payment_hash_1).unwrap();
1334 check_added_monitors!(nodes[0], 1);
1336 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1337 assert_eq!(events.len(), 1);
1338 let payment_event = SendEvent::from_event(events.pop().unwrap());
1339 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1340 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1341 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1342 check_added_monitors!(nodes[1], 1);
1343 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1345 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1346 check_added_monitors!(nodes[0], 1);
1347 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1348 check_added_monitors!(nodes[0], 1);
1350 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1352 // Route the second payment, generating an update_add_htlc/commitment_signed
1353 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1354 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1355 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1356 check_added_monitors!(nodes[0], 1);
1357 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1358 assert_eq!(events.len(), 1);
1359 let payment_event = SendEvent::from_event(events.pop().unwrap());
1360 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1362 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1364 // Deliver the final RAA for the first payment, which does not require a response. RAAs
1365 // generally require a commitment_signed, so the fact that we're expecting an opposite response
1366 // to the next message also tests resetting the delivery order.
1367 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1368 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1369 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1370 check_added_monitors!(nodes[1], 1);
1372 // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1373 // RAA/CS response, which should be generated when we call test_restore_channel_monitor (with
1374 // the appropriate HTLC acceptance).
1375 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1376 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1377 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1378 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1380 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1381 nodes[1].node.test_restore_channel_monitor();
1382 check_added_monitors!(nodes[1], 1);
1384 expect_pending_htlcs_forwardable!(nodes[1]);
1385 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1387 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1388 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1389 check_added_monitors!(nodes[0], 1);
1390 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1391 check_added_monitors!(nodes[0], 1);
1393 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1394 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1395 check_added_monitors!(nodes[1], 1);
1397 expect_pending_htlcs_forwardable!(nodes[1]);
1398 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1400 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1401 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1405 fn test_monitor_update_fail_claim() {
1406 // Basic test for monitor update failures when processing claim_funds calls.
1407 // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1408 // update to claim the payment. We then send a payment C->B->A, making the forward of this
1409 // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1410 // updating and claim the payment on B.
1411 let chanmon_cfgs = create_chanmon_cfgs(3);
1412 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1413 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1414 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1415 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1416 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1418 // Rebalance a bit so that we can send backwards from 3 to 2.
1419 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1421 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1423 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1424 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1425 check_added_monitors!(nodes[1], 1);
1427 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1428 let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1429 nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1430 check_added_monitors!(nodes[2], 1);
1432 // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1433 // paused, so forward shouldn't succeed until we call test_restore_channel_monitor().
1434 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1436 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1437 assert_eq!(events.len(), 1);
1438 let payment_event = SendEvent::from_event(events.pop().unwrap());
1439 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1440 let events = nodes[1].node.get_and_clear_pending_msg_events();
1441 assert_eq!(events.len(), 0);
1442 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1443 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1445 let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1446 nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1447 commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1449 let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1450 assert_eq!(msg_events.len(), 1);
1451 match msg_events[0] {
1452 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1453 assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1454 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1456 _ => panic!("Unexpected event"),
1459 let events = nodes[2].node.get_and_clear_pending_events();
1460 assert_eq!(events.len(), 1);
1461 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1462 assert_eq!(payment_hash, payment_hash_2);
1463 assert!(!rejected_by_dest);
1464 } else { panic!("Unexpected event!"); }
1466 // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1467 nodes[1].node.test_restore_channel_monitor();
1468 check_added_monitors!(nodes[1], 1);
1470 let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1471 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1472 commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1474 let events = nodes[0].node.get_and_clear_pending_events();
1475 assert_eq!(events.len(), 1);
1476 if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1477 assert_eq!(payment_preimage, payment_preimage_1);
1478 } else { panic!("Unexpected event!"); }
1482 fn test_monitor_update_on_pending_forwards() {
1483 // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1484 // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1485 // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1486 // from C to A will be pending a forward to A.
1487 let chanmon_cfgs = create_chanmon_cfgs(3);
1488 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1489 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1490 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1491 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1492 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::supported(), InitFeatures::supported());
1494 // Rebalance a bit so that we can send backwards from 3 to 1.
1495 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1497 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1498 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1));
1499 expect_pending_htlcs_forwardable!(nodes[2]);
1500 check_added_monitors!(nodes[2], 1);
1502 let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1503 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1504 commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1505 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1507 let route = nodes[2].router.get_route(&nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1508 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1509 nodes[2].node.send_payment(route, payment_hash_2).unwrap();
1510 check_added_monitors!(nodes[2], 1);
1512 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1513 assert_eq!(events.len(), 1);
1514 let payment_event = SendEvent::from_event(events.pop().unwrap());
1515 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1516 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1518 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1519 expect_pending_htlcs_forwardable!(nodes[1]);
1520 check_added_monitors!(nodes[1], 1);
1521 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1522 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1524 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1525 nodes[1].node.test_restore_channel_monitor();
1526 check_added_monitors!(nodes[1], 1);
1528 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1529 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1530 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1531 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1533 let events = nodes[0].node.get_and_clear_pending_events();
1534 assert_eq!(events.len(), 2);
1535 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1536 assert_eq!(payment_hash, payment_hash_1);
1537 assert!(rejected_by_dest);
1538 } else { panic!("Unexpected event!"); }
1540 Event::PendingHTLCsForwardable { .. } => { },
1541 _ => panic!("Unexpected event"),
1543 nodes[0].node.process_pending_htlc_forwards();
1544 expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1546 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1550 fn monitor_update_claim_fail_no_response() {
1551 // Test for claim_funds resulting in both a monitor update failure and no message response (due
1552 // to channel being AwaitingRAA).
1553 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1555 let chanmon_cfgs = create_chanmon_cfgs(2);
1556 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1557 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1558 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1559 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::supported(), InitFeatures::supported());
1561 // Forward a payment for B to claim
1562 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1564 // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1565 let route = nodes[0].router.get_route(&nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV).unwrap();
1566 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1567 nodes[0].node.send_payment(route, payment_hash_2).unwrap();
1568 check_added_monitors!(nodes[0], 1);
1570 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1571 assert_eq!(events.len(), 1);
1572 let payment_event = SendEvent::from_event(events.pop().unwrap());
1573 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1574 let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1576 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1577 assert!(nodes[1].node.claim_funds(payment_preimage_1, 1_000_000));
1578 check_added_monitors!(nodes[1], 1);
1579 let events = nodes[1].node.get_and_clear_pending_msg_events();
1580 assert_eq!(events.len(), 0);
1581 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1583 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1584 nodes[1].node.test_restore_channel_monitor();
1585 check_added_monitors!(nodes[1], 1);
1586 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1588 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1589 check_added_monitors!(nodes[1], 1);
1590 expect_pending_htlcs_forwardable!(nodes[1]);
1591 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1593 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1594 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1595 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1597 let events = nodes[0].node.get_and_clear_pending_events();
1598 assert_eq!(events.len(), 1);
1600 Event::PaymentSent { ref payment_preimage } => {
1601 assert_eq!(*payment_preimage, payment_preimage_1);
1603 _ => panic!("Unexpected event"),
1606 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1609 // Note that restore_between_fails with !fail_on_generate is useless
1610 // Also note that !fail_on_generate && !fail_on_signed is useless
1611 // Finally, note that !fail_on_signed is not possible with fail_on_generate && !restore_between_fails
1612 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1613 // restore_b_before_conf has no meaning if !confirm_a_first
1614 fn do_during_funding_monitor_fail(fail_on_generate: bool, restore_between_fails: bool, fail_on_signed: bool, confirm_a_first: bool, restore_b_before_conf: bool) {
1615 // Test that if the monitor update generated by funding_transaction_generated fails we continue
1616 // the channel setup happily after the update is restored.
1617 let chanmon_cfgs = create_chanmon_cfgs(2);
1618 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1619 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1620 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1622 nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43).unwrap();
1623 nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1624 nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::supported(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1626 let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1628 if fail_on_generate {
1629 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1631 nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1632 check_added_monitors!(nodes[0], 1);
1634 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1635 nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id()));
1636 check_added_monitors!(nodes[1], 1);
1638 if restore_between_fails {
1639 assert!(fail_on_generate);
1640 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1641 nodes[0].node.test_restore_channel_monitor();
1642 check_added_monitors!(nodes[0], 1);
1643 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1644 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1648 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1650 assert!(restore_between_fails || !fail_on_generate); // We can't switch to good now (there's no monitor update)
1651 assert!(fail_on_generate); // Somebody has to fail
1653 nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1654 if fail_on_signed || !restore_between_fails {
1655 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1656 if fail_on_generate && !restore_between_fails {
1657 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented funding_signed from allowing funding broadcast".to_string(), 1);
1658 check_added_monitors!(nodes[0], 0);
1660 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1661 check_added_monitors!(nodes[0], 1);
1663 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1664 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1665 nodes[0].node.test_restore_channel_monitor();
1668 check_added_monitors!(nodes[0], 1);
1670 let events = nodes[0].node.get_and_clear_pending_events();
1671 assert_eq!(events.len(), 1);
1673 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1674 assert_eq!(user_channel_id, 43);
1675 assert_eq!(*funding_txo, funding_output);
1677 _ => panic!("Unexpected event"),
1680 if confirm_a_first {
1681 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1682 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1684 assert!(!restore_b_before_conf);
1685 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1686 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1689 // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1690 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1691 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1692 reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1693 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1694 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1696 if !restore_b_before_conf {
1697 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1698 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1699 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1702 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1703 nodes[1].node.test_restore_channel_monitor();
1704 check_added_monitors!(nodes[1], 1);
1706 let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1707 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1709 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1710 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1711 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1713 if restore_b_before_conf {
1714 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1716 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1717 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1719 for node in nodes.iter() {
1720 assert!(node.router.handle_channel_announcement(&announcement).unwrap());
1721 node.router.handle_channel_update(&as_update).unwrap();
1722 node.router.handle_channel_update(&bs_update).unwrap();
1725 send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1726 close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1730 fn during_funding_monitor_fail() {
1731 do_during_funding_monitor_fail(false, false, true, true, true);
1732 do_during_funding_monitor_fail(true, false, true, false, false);
1733 do_during_funding_monitor_fail(true, true, true, true, false);
1734 do_during_funding_monitor_fail(true, true, false, false, false);