1 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
3 //! There are a bunch of these as their handling is relatively error-prone so they are split out
4 //! here. See also the chanmon_fail_consistency fuzz test.
6 use chain::transaction::OutPoint;
7 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure};
8 use ln::channelmonitor::ChannelMonitorUpdateErr;
9 use ln::features::InitFeatures;
11 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
12 use routing::router::get_route;
13 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
14 use util::errors::APIError;
16 use bitcoin::hashes::sha256::Hash as Sha256;
17 use bitcoin::hashes::Hash;
19 use ln::functional_test_utils::*;
24 fn test_simple_monitor_permanent_update_fail() {
25 // Test that we handle a simple permanent monitor update failure
26 let chanmon_cfgs = create_chanmon_cfgs(2);
27 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
28 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
29 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
30 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
31 let logger = test_utils::TestLogger::new();
33 let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
35 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
36 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
37 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
38 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), true, APIError::ChannelUnavailable {..}, {});
39 check_added_monitors!(nodes[0], 2);
41 let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
42 assert_eq!(events_1.len(), 2);
44 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
45 _ => panic!("Unexpected event"),
48 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
49 _ => panic!("Unexpected event"),
52 // TODO: Once we hit the chain with the failure transaction we should check that we get a
53 // PaymentFailed event
55 assert_eq!(nodes[0].node.list_channels().len(), 0);
58 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
59 // Test that we can recover from a simple temporary monitor update failure optionally with
60 // a disconnect in between
61 let chanmon_cfgs = create_chanmon_cfgs(2);
62 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
63 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
64 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
65 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
66 let logger = test_utils::TestLogger::new();
68 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
70 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
73 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
74 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
75 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), false, APIError::MonitorUpdateFailed, {});
76 check_added_monitors!(nodes[0], 1);
79 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
80 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
81 assert_eq!(nodes[0].node.list_channels().len(), 1);
84 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
85 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
86 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
89 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
90 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
91 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
92 check_added_monitors!(nodes[0], 0);
94 let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
95 assert_eq!(events_2.len(), 1);
96 let payment_event = SendEvent::from_event(events_2.pop().unwrap());
97 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
98 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
99 commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
101 expect_pending_htlcs_forwardable!(nodes[1]);
103 let events_3 = nodes[1].node.get_and_clear_pending_events();
104 assert_eq!(events_3.len(), 1);
106 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
107 assert_eq!(payment_hash_1, *payment_hash);
108 assert_eq!(*payment_secret, None);
109 assert_eq!(amt, 1000000);
111 _ => panic!("Unexpected event"),
114 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
116 // Now set it to failed again...
117 let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
119 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
120 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
121 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
122 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
123 check_added_monitors!(nodes[0], 1);
126 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
127 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
128 assert_eq!(nodes[0].node.list_channels().len(), 1);
131 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
132 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
133 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
136 // ...and make sure we can force-close a frozen channel
137 nodes[0].node.force_close_channel(&channel_id);
138 check_added_monitors!(nodes[0], 1);
139 check_closed_broadcast!(nodes[0], false);
141 // TODO: Once we hit the chain with the failure transaction we should check that we get a
142 // PaymentFailed event
144 assert_eq!(nodes[0].node.list_channels().len(), 0);
148 fn test_simple_monitor_temporary_update_fail() {
149 do_test_simple_monitor_temporary_update_fail(false);
150 do_test_simple_monitor_temporary_update_fail(true);
153 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
154 let disconnect_flags = 8 | 16;
156 // Test that we can recover from a temporary monitor update failure with some in-flight
157 // HTLCs going on at the same time potentially with some disconnection thrown in.
158 // * First we route a payment, then get a temporary monitor update failure when trying to
159 // route a second payment. We then claim the first payment.
160 // * If disconnect_count is set, we will disconnect at this point (which is likely as
161 // TemporaryFailure likely indicates net disconnect which resulted in failing to update
162 // the ChannelMonitor on a watchtower).
163 // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
164 // immediately, otherwise we wait disconnect and deliver them via the reconnect
165 // channel_reestablish processing (ie disconnect_count & 16 makes no sense if
166 // disconnect_count & !disconnect_flags is 0).
167 // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
168 // through message sending, potentially disconnect/reconnecting multiple times based on
169 // disconnect_count, to get the update_fulfill_htlc through.
170 // * We then walk through more message exchanges to get the original update_add_htlc
171 // through, swapping message ordering based on disconnect_count & 8 and optionally
172 // disconnect/reconnecting based on disconnect_count.
173 let chanmon_cfgs = create_chanmon_cfgs(2);
174 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
175 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
176 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
177 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
178 let logger = test_utils::TestLogger::new();
180 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
182 // Now try to send a second payment which will fail to send
183 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
185 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
186 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
187 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
188 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
189 check_added_monitors!(nodes[0], 1);
192 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
193 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
194 assert_eq!(nodes[0].node.list_channels().len(), 1);
196 // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
197 // but nodes[0] won't respond since it is frozen.
198 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
199 check_added_monitors!(nodes[1], 1);
200 let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
201 assert_eq!(events_2.len(), 1);
202 let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
203 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
204 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
205 assert!(update_add_htlcs.is_empty());
206 assert_eq!(update_fulfill_htlcs.len(), 1);
207 assert!(update_fail_htlcs.is_empty());
208 assert!(update_fail_malformed_htlcs.is_empty());
209 assert!(update_fee.is_none());
211 if (disconnect_count & 16) == 0 {
212 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
213 let events_3 = nodes[0].node.get_and_clear_pending_events();
214 assert_eq!(events_3.len(), 1);
216 Event::PaymentSent { ref payment_preimage } => {
217 assert_eq!(*payment_preimage, payment_preimage_1);
219 _ => panic!("Unexpected event"),
222 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
223 check_added_monitors!(nodes[0], 1);
224 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
225 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
228 (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
230 _ => panic!("Unexpected event"),
233 if disconnect_count & !disconnect_flags > 0 {
234 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
235 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
238 // Now fix monitor updating...
239 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
240 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
241 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
242 check_added_monitors!(nodes[0], 0);
244 macro_rules! disconnect_reconnect_peers { () => { {
245 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
246 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
248 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
249 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
250 assert_eq!(reestablish_1.len(), 1);
251 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
252 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
253 assert_eq!(reestablish_2.len(), 1);
255 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
256 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
257 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
258 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
260 assert!(as_resp.0.is_none());
261 assert!(bs_resp.0.is_none());
263 (reestablish_1, reestablish_2, as_resp, bs_resp)
266 let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
267 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
268 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
270 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
271 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
272 assert_eq!(reestablish_1.len(), 1);
273 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
274 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
275 assert_eq!(reestablish_2.len(), 1);
277 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
278 check_added_monitors!(nodes[0], 0);
279 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
280 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
281 check_added_monitors!(nodes[1], 0);
282 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
284 assert!(as_resp.0.is_none());
285 assert!(bs_resp.0.is_none());
287 assert!(bs_resp.1.is_none());
288 if (disconnect_count & 16) == 0 {
289 assert!(bs_resp.2.is_none());
291 assert!(as_resp.1.is_some());
292 assert!(as_resp.2.is_some());
293 assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
295 assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
296 assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
297 assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
298 assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
299 assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
300 assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
302 assert!(as_resp.1.is_none());
304 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
305 let events_3 = nodes[0].node.get_and_clear_pending_events();
306 assert_eq!(events_3.len(), 1);
308 Event::PaymentSent { ref payment_preimage } => {
309 assert_eq!(*payment_preimage, payment_preimage_1);
311 _ => panic!("Unexpected event"),
314 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
315 let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
316 // No commitment_signed so get_event_msg's assert(len == 1) passes
317 check_added_monitors!(nodes[0], 1);
319 as_resp.1 = Some(as_resp_raa);
323 if disconnect_count & !disconnect_flags > 1 {
324 let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
326 if (disconnect_count & 16) == 0 {
327 assert!(reestablish_1 == second_reestablish_1);
328 assert!(reestablish_2 == second_reestablish_2);
330 assert!(as_resp == second_as_resp);
331 assert!(bs_resp == second_bs_resp);
334 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
336 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
337 assert_eq!(events_4.len(), 2);
338 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
339 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
340 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
343 _ => panic!("Unexpected event"),
347 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
349 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
350 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
351 let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
352 // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
353 check_added_monitors!(nodes[1], 1);
355 if disconnect_count & !disconnect_flags > 2 {
356 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
358 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
359 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
361 assert!(as_resp.2.is_none());
362 assert!(bs_resp.2.is_none());
365 let as_commitment_update;
366 let bs_second_commitment_update;
368 macro_rules! handle_bs_raa { () => {
369 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
370 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
371 assert!(as_commitment_update.update_add_htlcs.is_empty());
372 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
373 assert!(as_commitment_update.update_fail_htlcs.is_empty());
374 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
375 assert!(as_commitment_update.update_fee.is_none());
376 check_added_monitors!(nodes[0], 1);
379 macro_rules! handle_initial_raa { () => {
380 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
381 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
382 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
383 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
384 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
385 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
386 assert!(bs_second_commitment_update.update_fee.is_none());
387 check_added_monitors!(nodes[1], 1);
390 if (disconnect_count & 8) == 0 {
393 if disconnect_count & !disconnect_flags > 3 {
394 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
396 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
397 assert!(bs_resp.1.is_none());
399 assert!(as_resp.2.unwrap() == as_commitment_update);
400 assert!(bs_resp.2.is_none());
402 assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
405 handle_initial_raa!();
407 if disconnect_count & !disconnect_flags > 4 {
408 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
410 assert!(as_resp.1.is_none());
411 assert!(bs_resp.1.is_none());
413 assert!(as_resp.2.unwrap() == as_commitment_update);
414 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
417 handle_initial_raa!();
419 if disconnect_count & !disconnect_flags > 3 {
420 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
422 assert!(as_resp.1.is_none());
423 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
425 assert!(as_resp.2.is_none());
426 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
428 assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
433 if disconnect_count & !disconnect_flags > 4 {
434 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
436 assert!(as_resp.1.is_none());
437 assert!(bs_resp.1.is_none());
439 assert!(as_resp.2.unwrap() == as_commitment_update);
440 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
444 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
445 let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
446 // No commitment_signed so get_event_msg's assert(len == 1) passes
447 check_added_monitors!(nodes[0], 1);
449 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
450 let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
451 // No commitment_signed so get_event_msg's assert(len == 1) passes
452 check_added_monitors!(nodes[1], 1);
454 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
455 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
456 check_added_monitors!(nodes[1], 1);
458 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
459 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
460 check_added_monitors!(nodes[0], 1);
462 expect_pending_htlcs_forwardable!(nodes[1]);
464 let events_5 = nodes[1].node.get_and_clear_pending_events();
465 assert_eq!(events_5.len(), 1);
467 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
468 assert_eq!(payment_hash_2, *payment_hash);
469 assert_eq!(*payment_secret, None);
470 assert_eq!(amt, 1000000);
472 _ => panic!("Unexpected event"),
475 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
479 fn test_monitor_temporary_update_fail_a() {
480 do_test_monitor_temporary_update_fail(0);
481 do_test_monitor_temporary_update_fail(1);
482 do_test_monitor_temporary_update_fail(2);
483 do_test_monitor_temporary_update_fail(3);
484 do_test_monitor_temporary_update_fail(4);
485 do_test_monitor_temporary_update_fail(5);
489 fn test_monitor_temporary_update_fail_b() {
490 do_test_monitor_temporary_update_fail(2 | 8);
491 do_test_monitor_temporary_update_fail(3 | 8);
492 do_test_monitor_temporary_update_fail(4 | 8);
493 do_test_monitor_temporary_update_fail(5 | 8);
497 fn test_monitor_temporary_update_fail_c() {
498 do_test_monitor_temporary_update_fail(1 | 16);
499 do_test_monitor_temporary_update_fail(2 | 16);
500 do_test_monitor_temporary_update_fail(3 | 16);
501 do_test_monitor_temporary_update_fail(2 | 8 | 16);
502 do_test_monitor_temporary_update_fail(3 | 8 | 16);
506 fn test_monitor_update_fail_cs() {
507 // Tests handling of a monitor update failure when processing an incoming commitment_signed
508 let chanmon_cfgs = create_chanmon_cfgs(2);
509 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
510 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
511 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
512 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
513 let logger = test_utils::TestLogger::new();
515 let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
517 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
518 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
519 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
520 check_added_monitors!(nodes[0], 1);
523 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
524 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
526 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
527 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
528 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
529 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
530 check_added_monitors!(nodes[1], 1);
531 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
533 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
534 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
535 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
536 check_added_monitors!(nodes[1], 0);
537 let responses = nodes[1].node.get_and_clear_pending_msg_events();
538 assert_eq!(responses.len(), 2);
541 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
542 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
543 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
544 check_added_monitors!(nodes[0], 1);
546 _ => panic!("Unexpected event"),
549 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
550 assert!(updates.update_add_htlcs.is_empty());
551 assert!(updates.update_fulfill_htlcs.is_empty());
552 assert!(updates.update_fail_htlcs.is_empty());
553 assert!(updates.update_fail_malformed_htlcs.is_empty());
554 assert!(updates.update_fee.is_none());
555 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
557 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
558 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
559 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
560 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
561 check_added_monitors!(nodes[0], 1);
562 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
564 _ => panic!("Unexpected event"),
567 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
568 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
569 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
570 check_added_monitors!(nodes[0], 0);
572 let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
573 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
574 check_added_monitors!(nodes[1], 1);
576 expect_pending_htlcs_forwardable!(nodes[1]);
578 let events = nodes[1].node.get_and_clear_pending_events();
579 assert_eq!(events.len(), 1);
581 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
582 assert_eq!(payment_hash, our_payment_hash);
583 assert_eq!(payment_secret, None);
584 assert_eq!(amt, 1000000);
586 _ => panic!("Unexpected event"),
589 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
593 fn test_monitor_update_fail_no_rebroadcast() {
594 // Tests handling of a monitor update failure when no message rebroadcasting on
595 // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
597 let chanmon_cfgs = create_chanmon_cfgs(2);
598 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
599 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
600 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
601 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
602 let logger = test_utils::TestLogger::new();
604 let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
606 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
607 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
608 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
609 check_added_monitors!(nodes[0], 1);
612 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
613 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
614 let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
616 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
617 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
618 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
619 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
620 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
621 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
622 check_added_monitors!(nodes[1], 1);
624 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
625 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
626 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
627 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
628 check_added_monitors!(nodes[1], 0);
629 expect_pending_htlcs_forwardable!(nodes[1]);
631 let events = nodes[1].node.get_and_clear_pending_events();
632 assert_eq!(events.len(), 1);
634 Event::PaymentReceived { payment_hash, .. } => {
635 assert_eq!(payment_hash, our_payment_hash);
637 _ => panic!("Unexpected event"),
640 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
644 fn test_monitor_update_raa_while_paused() {
645 // Tests handling of an RAA while monitor updating has already been marked failed.
646 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
647 let chanmon_cfgs = create_chanmon_cfgs(2);
648 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
649 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
650 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
651 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
652 let logger = test_utils::TestLogger::new();
654 send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
655 let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
657 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
658 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
659 nodes[0].node.send_payment(&route, our_payment_hash_1, &None).unwrap();
660 check_added_monitors!(nodes[0], 1);
662 let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
664 let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
666 let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
667 let route = get_route(&nodes[1].node.get_our_node_id(), net_graph_msg_handler, &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
668 nodes[1].node.send_payment(&route, our_payment_hash_2, &None).unwrap();
669 check_added_monitors!(nodes[1], 1);
671 let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
673 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
674 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
675 check_added_monitors!(nodes[1], 1);
676 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
678 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
679 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
680 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
681 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
682 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
683 check_added_monitors!(nodes[0], 1);
685 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
686 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
687 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
688 check_added_monitors!(nodes[0], 1);
690 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
691 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
692 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
693 check_added_monitors!(nodes[0], 0);
695 let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
696 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
697 check_added_monitors!(nodes[1], 1);
698 let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
700 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
701 check_added_monitors!(nodes[1], 1);
702 let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
704 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
705 check_added_monitors!(nodes[0], 1);
706 let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
708 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
709 check_added_monitors!(nodes[0], 1);
710 expect_pending_htlcs_forwardable!(nodes[0]);
711 expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
713 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
714 check_added_monitors!(nodes[1], 1);
715 expect_pending_htlcs_forwardable!(nodes[1]);
716 expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
718 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
719 claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
722 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
723 // Tests handling of a monitor update failure when processing an incoming RAA
724 let chanmon_cfgs = create_chanmon_cfgs(3);
725 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
726 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
727 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
728 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
729 let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
730 let logger = test_utils::TestLogger::new();
732 // Rebalance a bit so that we can send backwards from 2 to 1.
733 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
735 // Route a first payment that we'll fail backwards
736 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
738 // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
739 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
740 expect_pending_htlcs_forwardable!(nodes[2]);
741 check_added_monitors!(nodes[2], 1);
743 let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
744 assert!(updates.update_add_htlcs.is_empty());
745 assert!(updates.update_fulfill_htlcs.is_empty());
746 assert_eq!(updates.update_fail_htlcs.len(), 1);
747 assert!(updates.update_fail_malformed_htlcs.is_empty());
748 assert!(updates.update_fee.is_none());
749 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
751 let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
752 check_added_monitors!(nodes[0], 0);
754 // While the second channel is AwaitingRAA, forward a second payment to get it into the
756 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
758 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
759 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
760 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
761 check_added_monitors!(nodes[0], 1);
764 let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
765 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
766 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
768 expect_pending_htlcs_forwardable!(nodes[1]);
769 check_added_monitors!(nodes[1], 0);
770 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
772 // Now fail monitor updating.
773 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
774 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
775 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
776 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
777 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
778 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
779 check_added_monitors!(nodes[1], 1);
781 // Attempt to forward a third payment but fail due to the second channel being unavailable
783 let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
785 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
786 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
787 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
788 check_added_monitors!(nodes[0], 1);
791 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
792 send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
793 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
794 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
795 check_added_monitors!(nodes[1], 0);
797 let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
798 assert_eq!(events_2.len(), 1);
799 match events_2.remove(0) {
800 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
801 assert_eq!(node_id, nodes[0].node.get_our_node_id());
802 assert!(updates.update_fulfill_htlcs.is_empty());
803 assert_eq!(updates.update_fail_htlcs.len(), 1);
804 assert!(updates.update_fail_malformed_htlcs.is_empty());
805 assert!(updates.update_add_htlcs.is_empty());
806 assert!(updates.update_fee.is_none());
808 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
809 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
811 let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
812 assert_eq!(msg_events.len(), 1);
813 match msg_events[0] {
814 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
815 assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
816 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
818 _ => panic!("Unexpected event"),
821 let events = nodes[0].node.get_and_clear_pending_events();
822 assert_eq!(events.len(), 1);
823 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
824 assert_eq!(payment_hash, payment_hash_3);
825 assert!(!rejected_by_dest);
826 } else { panic!("Unexpected event!"); }
828 _ => panic!("Unexpected event type!"),
831 let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
832 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
833 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
834 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
835 let route = get_route(&nodes[2].node.get_our_node_id(), net_graph_msg_handler, &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
836 nodes[2].node.send_payment(&route, payment_hash_4, &None).unwrap();
837 check_added_monitors!(nodes[2], 1);
839 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
840 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
841 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
842 check_added_monitors!(nodes[1], 1);
843 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
844 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
845 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
846 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
847 (Some(payment_preimage_4), Some(payment_hash_4))
848 } else { (None, None) };
850 // Restore monitor updating, ensuring we immediately get a fail-back update and a
851 // update_add update.
852 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
853 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
854 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
855 check_added_monitors!(nodes[1], 0);
856 expect_pending_htlcs_forwardable!(nodes[1]);
857 check_added_monitors!(nodes[1], 1);
859 let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
860 if test_ignore_second_cs {
861 assert_eq!(events_3.len(), 3);
863 assert_eq!(events_3.len(), 2);
866 // Note that the ordering of the events for different nodes is non-prescriptive, though the
867 // ordering of the two events that both go to nodes[2] have to stay in the same order.
868 let messages_a = match events_3.pop().unwrap() {
869 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
870 assert_eq!(node_id, nodes[0].node.get_our_node_id());
871 assert!(updates.update_fulfill_htlcs.is_empty());
872 assert_eq!(updates.update_fail_htlcs.len(), 1);
873 assert!(updates.update_fail_malformed_htlcs.is_empty());
874 assert!(updates.update_add_htlcs.is_empty());
875 assert!(updates.update_fee.is_none());
876 (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
878 _ => panic!("Unexpected event type!"),
880 let raa = if test_ignore_second_cs {
881 match events_3.remove(1) {
882 MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
883 assert_eq!(node_id, nodes[2].node.get_our_node_id());
886 _ => panic!("Unexpected event"),
889 let send_event_b = SendEvent::from_event(events_3.remove(0));
890 assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
892 // Now deliver the new messages...
894 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
895 commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
896 let events_4 = nodes[0].node.get_and_clear_pending_events();
897 assert_eq!(events_4.len(), 1);
898 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
899 assert_eq!(payment_hash, payment_hash_1);
900 assert!(rejected_by_dest);
901 } else { panic!("Unexpected event!"); }
903 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
904 if test_ignore_second_cs {
905 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
906 check_added_monitors!(nodes[2], 1);
907 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
908 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
909 check_added_monitors!(nodes[2], 1);
910 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
911 assert!(bs_cs.update_add_htlcs.is_empty());
912 assert!(bs_cs.update_fail_htlcs.is_empty());
913 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
914 assert!(bs_cs.update_fulfill_htlcs.is_empty());
915 assert!(bs_cs.update_fee.is_none());
917 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
918 check_added_monitors!(nodes[1], 1);
919 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
920 assert!(as_cs.update_add_htlcs.is_empty());
921 assert!(as_cs.update_fail_htlcs.is_empty());
922 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
923 assert!(as_cs.update_fulfill_htlcs.is_empty());
924 assert!(as_cs.update_fee.is_none());
926 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
927 check_added_monitors!(nodes[1], 1);
928 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
930 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
931 check_added_monitors!(nodes[2], 1);
932 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
934 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
935 check_added_monitors!(nodes[2], 1);
936 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
938 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
939 check_added_monitors!(nodes[1], 1);
940 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
942 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
945 expect_pending_htlcs_forwardable!(nodes[2]);
947 let events_6 = nodes[2].node.get_and_clear_pending_events();
948 assert_eq!(events_6.len(), 1);
950 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
951 _ => panic!("Unexpected event"),
954 if test_ignore_second_cs {
955 expect_pending_htlcs_forwardable!(nodes[1]);
956 check_added_monitors!(nodes[1], 1);
958 send_event = SendEvent::from_node(&nodes[1]);
959 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
960 assert_eq!(send_event.msgs.len(), 1);
961 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
962 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
964 expect_pending_htlcs_forwardable!(nodes[0]);
966 let events_9 = nodes[0].node.get_and_clear_pending_events();
967 assert_eq!(events_9.len(), 1);
969 Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
970 _ => panic!("Unexpected event"),
972 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
975 claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
979 fn test_monitor_update_fail_raa() {
980 do_test_monitor_update_fail_raa(false);
981 do_test_monitor_update_fail_raa(true);
985 fn test_monitor_update_fail_reestablish() {
986 // Simple test for message retransmission after monitor update failure on
987 // channel_reestablish generating a monitor update (which comes from freeing holding cell
989 let chanmon_cfgs = create_chanmon_cfgs(3);
990 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
991 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
992 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
993 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
994 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
996 let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
998 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
999 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1001 assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
1002 check_added_monitors!(nodes[2], 1);
1003 let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1004 assert!(updates.update_add_htlcs.is_empty());
1005 assert!(updates.update_fail_htlcs.is_empty());
1006 assert!(updates.update_fail_malformed_htlcs.is_empty());
1007 assert!(updates.update_fee.is_none());
1008 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1009 nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1010 check_added_monitors!(nodes[1], 1);
1011 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1012 commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
1014 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1015 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1016 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1018 let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1019 let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1021 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1023 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1024 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1025 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1026 check_added_monitors!(nodes[1], 1);
1028 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1029 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1031 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1032 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1034 assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
1035 assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
1037 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1039 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1040 check_added_monitors!(nodes[1], 0);
1041 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1043 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1044 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1045 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1046 check_added_monitors!(nodes[1], 0);
1048 updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1049 assert!(updates.update_add_htlcs.is_empty());
1050 assert!(updates.update_fail_htlcs.is_empty());
1051 assert!(updates.update_fail_malformed_htlcs.is_empty());
1052 assert!(updates.update_fee.is_none());
1053 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1054 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1055 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1057 let events = nodes[0].node.get_and_clear_pending_events();
1058 assert_eq!(events.len(), 1);
1060 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1061 _ => panic!("Unexpected event"),
1066 fn raa_no_response_awaiting_raa_state() {
1067 // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1068 // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1069 // in question (assuming it intends to respond with a CS after monitor updating is restored).
1070 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1071 let chanmon_cfgs = create_chanmon_cfgs(2);
1072 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1073 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1074 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1075 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1076 let logger = test_utils::TestLogger::new();
1078 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1079 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1080 let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1082 // Queue up two payments - one will be delivered right away, one immediately goes into the
1083 // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1084 // immediately after a CS. By setting failing the monitor update failure from the CS (which
1085 // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1086 // generation during RAA while in monitor-update-failed state.
1088 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1089 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1090 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1091 check_added_monitors!(nodes[0], 1);
1092 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1093 check_added_monitors!(nodes[0], 0);
1096 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1097 assert_eq!(events.len(), 1);
1098 let payment_event = SendEvent::from_event(events.pop().unwrap());
1099 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1100 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1101 check_added_monitors!(nodes[1], 1);
1103 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1104 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1105 check_added_monitors!(nodes[0], 1);
1106 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1107 assert_eq!(events.len(), 1);
1108 let payment_event = SendEvent::from_event(events.pop().unwrap());
1110 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1111 check_added_monitors!(nodes[0], 1);
1112 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1114 // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1115 // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1116 // then restore channel monitor updates.
1117 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1118 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1119 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1120 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1121 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1122 check_added_monitors!(nodes[1], 1);
1124 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1125 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1126 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1127 check_added_monitors!(nodes[1], 1);
1129 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1130 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1131 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1132 // nodes[1] should be AwaitingRAA here!
1133 check_added_monitors!(nodes[1], 0);
1134 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1135 expect_pending_htlcs_forwardable!(nodes[1]);
1136 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1138 // We send a third payment here, which is somewhat of a redundant test, but the
1139 // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1140 // commitment transaction states) whereas here we can explicitly check for it.
1142 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1143 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1144 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
1145 check_added_monitors!(nodes[0], 0);
1146 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1148 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1149 check_added_monitors!(nodes[0], 1);
1150 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1151 assert_eq!(events.len(), 1);
1152 let payment_event = SendEvent::from_event(events.pop().unwrap());
1154 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1155 check_added_monitors!(nodes[0], 1);
1156 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1158 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1159 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1160 check_added_monitors!(nodes[1], 1);
1161 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1163 // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1164 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1165 check_added_monitors!(nodes[1], 1);
1166 expect_pending_htlcs_forwardable!(nodes[1]);
1167 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1168 let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1170 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1171 check_added_monitors!(nodes[0], 1);
1173 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1174 check_added_monitors!(nodes[0], 1);
1175 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1177 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1178 check_added_monitors!(nodes[1], 1);
1179 expect_pending_htlcs_forwardable!(nodes[1]);
1180 expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1182 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1183 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1184 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1188 fn claim_while_disconnected_monitor_update_fail() {
1189 // Test for claiming a payment while disconnected and then having the resulting
1190 // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1191 // contrived case for nodes with network instability.
1192 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1193 // code introduced a regression in this test (specifically, this caught a removal of the
1194 // channel_reestablish handling ensuring the order was sensical given the messages used).
1195 let chanmon_cfgs = create_chanmon_cfgs(2);
1196 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1197 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1198 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1199 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1200 let logger = test_utils::TestLogger::new();
1202 // Forward a payment for B to claim
1203 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1205 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1206 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1208 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1209 check_added_monitors!(nodes[1], 1);
1211 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1212 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1214 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1215 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1217 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1218 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1220 // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1222 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1224 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1225 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1226 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1227 check_added_monitors!(nodes[1], 1);
1228 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1230 // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1231 // the monitor still failed
1232 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1234 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1235 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1236 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1237 check_added_monitors!(nodes[0], 1);
1240 let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1241 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1242 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1243 check_added_monitors!(nodes[1], 1);
1244 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1245 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1246 // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1247 // until we've channel_monitor_update'd and updated for the new commitment transaction.
1249 // Now un-fail the monitor, which will result in B sending its original commitment update,
1250 // receiving the commitment update from A, and the resulting commitment dances.
1251 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1252 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1253 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1254 check_added_monitors!(nodes[1], 0);
1256 let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1257 assert_eq!(bs_msgs.len(), 2);
1260 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1261 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1262 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1263 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1264 check_added_monitors!(nodes[0], 1);
1266 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1267 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1268 check_added_monitors!(nodes[1], 1);
1270 _ => panic!("Unexpected event"),
1274 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1275 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1276 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1277 check_added_monitors!(nodes[0], 1);
1279 _ => panic!("Unexpected event"),
1282 let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1284 let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1285 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1286 check_added_monitors!(nodes[0], 1);
1287 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1289 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1290 check_added_monitors!(nodes[1], 1);
1291 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1292 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1293 check_added_monitors!(nodes[1], 1);
1295 expect_pending_htlcs_forwardable!(nodes[1]);
1296 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1298 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1299 check_added_monitors!(nodes[0], 1);
1301 let events = nodes[0].node.get_and_clear_pending_events();
1302 assert_eq!(events.len(), 1);
1304 Event::PaymentSent { ref payment_preimage } => {
1305 assert_eq!(*payment_preimage, payment_preimage_1);
1307 _ => panic!("Unexpected event"),
1310 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1314 fn monitor_failed_no_reestablish_response() {
1315 // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1316 // response to a commitment_signed.
1317 // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1318 // debug_assert!() failure in channel_reestablish handling.
1319 let chanmon_cfgs = create_chanmon_cfgs(2);
1320 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1321 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1322 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1323 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1324 let logger = test_utils::TestLogger::new();
1326 // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1328 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1330 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1331 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1332 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1333 check_added_monitors!(nodes[0], 1);
1336 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1337 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1338 assert_eq!(events.len(), 1);
1339 let payment_event = SendEvent::from_event(events.pop().unwrap());
1340 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1341 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1342 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1343 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1344 check_added_monitors!(nodes[1], 1);
1346 // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1347 // is still failing to update monitors.
1348 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1349 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1351 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1352 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1354 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1355 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1357 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1358 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1360 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1361 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1362 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1363 check_added_monitors!(nodes[1], 0);
1364 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1366 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1367 check_added_monitors!(nodes[0], 1);
1368 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1369 check_added_monitors!(nodes[0], 1);
1371 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1372 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1373 check_added_monitors!(nodes[1], 1);
1375 expect_pending_htlcs_forwardable!(nodes[1]);
1376 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1378 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1382 fn first_message_on_recv_ordering() {
1383 // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1384 // messages, we're willing to flip the order of response messages if neccessary in resposne to
1385 // a commitment_signed which needs to send an RAA first.
1386 // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1387 // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1388 // response. To do this, we start routing two payments, with the final RAA for the first being
1389 // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1390 // have no pending response but will want to send a RAA/CS (with the updates for the second
1391 // payment applied).
1392 // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1393 let chanmon_cfgs = create_chanmon_cfgs(2);
1394 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1395 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1396 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1397 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1398 let logger = test_utils::TestLogger::new();
1400 // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1401 // can deliver it and fail the monitor update.
1402 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1404 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1405 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1406 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1407 check_added_monitors!(nodes[0], 1);
1410 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1411 assert_eq!(events.len(), 1);
1412 let payment_event = SendEvent::from_event(events.pop().unwrap());
1413 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1414 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1415 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1416 check_added_monitors!(nodes[1], 1);
1417 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1419 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1420 check_added_monitors!(nodes[0], 1);
1421 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1422 check_added_monitors!(nodes[0], 1);
1424 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1426 // Route the second payment, generating an update_add_htlc/commitment_signed
1427 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1429 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1430 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1431 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1432 check_added_monitors!(nodes[0], 1);
1434 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1435 assert_eq!(events.len(), 1);
1436 let payment_event = SendEvent::from_event(events.pop().unwrap());
1437 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1439 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1441 // Deliver the final RAA for the first payment, which does not require a response. RAAs
1442 // generally require a commitment_signed, so the fact that we're expecting an opposite response
1443 // to the next message also tests resetting the delivery order.
1444 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1445 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1446 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1447 check_added_monitors!(nodes[1], 1);
1449 // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1450 // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1451 // appropriate HTLC acceptance).
1452 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1453 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1454 check_added_monitors!(nodes[1], 1);
1455 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1456 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1458 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1459 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1460 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1461 check_added_monitors!(nodes[1], 0);
1463 expect_pending_htlcs_forwardable!(nodes[1]);
1464 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1466 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1467 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1468 check_added_monitors!(nodes[0], 1);
1469 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1470 check_added_monitors!(nodes[0], 1);
1472 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1473 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1474 check_added_monitors!(nodes[1], 1);
1476 expect_pending_htlcs_forwardable!(nodes[1]);
1477 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1479 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1480 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1484 fn test_monitor_update_fail_claim() {
1485 // Basic test for monitor update failures when processing claim_funds calls.
1486 // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1487 // update to claim the payment. We then send a payment C->B->A, making the forward of this
1488 // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1489 // updating and claim the payment on B.
1490 let chanmon_cfgs = create_chanmon_cfgs(3);
1491 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1492 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1493 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1494 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1495 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1496 let logger = test_utils::TestLogger::new();
1498 // Rebalance a bit so that we can send backwards from 3 to 2.
1499 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1501 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1503 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1504 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1505 check_added_monitors!(nodes[1], 1);
1507 let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1509 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1510 let route = get_route(&nodes[2].node.get_our_node_id(), net_graph_msg_handler, &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1511 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1512 check_added_monitors!(nodes[2], 1);
1515 // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1516 // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1517 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1519 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1520 assert_eq!(events.len(), 1);
1521 let payment_event = SendEvent::from_event(events.pop().unwrap());
1522 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1523 let events = nodes[1].node.get_and_clear_pending_msg_events();
1524 assert_eq!(events.len(), 0);
1525 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1526 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1528 let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1529 nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1530 commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1532 let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1533 assert_eq!(msg_events.len(), 1);
1534 match msg_events[0] {
1535 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1536 assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1537 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1539 _ => panic!("Unexpected event"),
1542 let events = nodes[2].node.get_and_clear_pending_events();
1543 assert_eq!(events.len(), 1);
1544 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1545 assert_eq!(payment_hash, payment_hash_2);
1546 assert!(!rejected_by_dest);
1547 } else { panic!("Unexpected event!"); }
1549 // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1550 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1551 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1552 check_added_monitors!(nodes[1], 0);
1554 let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1555 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1556 commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1558 let events = nodes[0].node.get_and_clear_pending_events();
1559 assert_eq!(events.len(), 1);
1560 if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1561 assert_eq!(payment_preimage, payment_preimage_1);
1562 } else { panic!("Unexpected event!"); }
1566 fn test_monitor_update_on_pending_forwards() {
1567 // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1568 // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1569 // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1570 // from C to A will be pending a forward to A.
1571 let chanmon_cfgs = create_chanmon_cfgs(3);
1572 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1573 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1574 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1575 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1576 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1577 let logger = test_utils::TestLogger::new();
1579 // Rebalance a bit so that we can send backwards from 3 to 1.
1580 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1582 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1583 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1584 expect_pending_htlcs_forwardable!(nodes[2]);
1585 check_added_monitors!(nodes[2], 1);
1587 let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1588 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1589 commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1590 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1592 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1594 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1595 let route = get_route(&nodes[2].node.get_our_node_id(), net_graph_msg_handler, &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1596 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1597 check_added_monitors!(nodes[2], 1);
1600 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1601 assert_eq!(events.len(), 1);
1602 let payment_event = SendEvent::from_event(events.pop().unwrap());
1603 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1604 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1606 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1607 expect_pending_htlcs_forwardable!(nodes[1]);
1608 check_added_monitors!(nodes[1], 1);
1609 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1610 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1612 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1613 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1614 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1615 check_added_monitors!(nodes[1], 0);
1617 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1618 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1619 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1620 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1622 let events = nodes[0].node.get_and_clear_pending_events();
1623 assert_eq!(events.len(), 2);
1624 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1625 assert_eq!(payment_hash, payment_hash_1);
1626 assert!(rejected_by_dest);
1627 } else { panic!("Unexpected event!"); }
1629 Event::PendingHTLCsForwardable { .. } => { },
1630 _ => panic!("Unexpected event"),
1632 nodes[0].node.process_pending_htlc_forwards();
1633 expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1635 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1639 fn monitor_update_claim_fail_no_response() {
1640 // Test for claim_funds resulting in both a monitor update failure and no message response (due
1641 // to channel being AwaitingRAA).
1642 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1644 let chanmon_cfgs = create_chanmon_cfgs(2);
1645 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1646 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1647 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1648 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1649 let logger = test_utils::TestLogger::new();
1651 // Forward a payment for B to claim
1652 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1654 // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1655 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1657 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1658 let route = get_route(&nodes[0].node.get_our_node_id(), net_graph_msg_handler, &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1659 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1660 check_added_monitors!(nodes[0], 1);
1663 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1664 assert_eq!(events.len(), 1);
1665 let payment_event = SendEvent::from_event(events.pop().unwrap());
1666 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1667 let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1669 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1670 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1671 check_added_monitors!(nodes[1], 1);
1672 let events = nodes[1].node.get_and_clear_pending_msg_events();
1673 assert_eq!(events.len(), 0);
1674 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1676 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1677 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1678 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1679 check_added_monitors!(nodes[1], 0);
1680 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1682 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1683 check_added_monitors!(nodes[1], 1);
1684 expect_pending_htlcs_forwardable!(nodes[1]);
1685 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1687 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1688 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1689 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1691 let events = nodes[0].node.get_and_clear_pending_events();
1692 assert_eq!(events.len(), 1);
1694 Event::PaymentSent { ref payment_preimage } => {
1695 assert_eq!(*payment_preimage, payment_preimage_1);
1697 _ => panic!("Unexpected event"),
1700 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1703 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1704 // restore_b_before_conf has no meaning if !confirm_a_first
1705 fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf: bool) {
1706 // Test that if the monitor update generated by funding_transaction_generated fails we continue
1707 // the channel setup happily after the update is restored.
1708 let chanmon_cfgs = create_chanmon_cfgs(2);
1709 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1710 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1711 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1713 nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1714 nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1715 nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1717 let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1719 nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1720 check_added_monitors!(nodes[0], 0);
1722 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1723 let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1724 let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1725 nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1726 check_added_monitors!(nodes[1], 1);
1728 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1729 nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1730 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1731 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1732 check_added_monitors!(nodes[0], 1);
1733 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1734 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1735 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1736 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1737 check_added_monitors!(nodes[0], 0);
1739 let events = nodes[0].node.get_and_clear_pending_events();
1740 assert_eq!(events.len(), 1);
1742 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1743 assert_eq!(user_channel_id, 43);
1744 assert_eq!(*funding_txo, funding_output);
1746 _ => panic!("Unexpected event"),
1749 if confirm_a_first {
1750 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1751 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1753 assert!(!restore_b_before_conf);
1754 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1755 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1758 // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1759 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1760 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1761 reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1762 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1763 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1765 if !restore_b_before_conf {
1766 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1767 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1768 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1771 *nodes[1].chan_monitor.update_ret.lock().unwrap() = Ok(());
1772 let (outpoint, latest_update) = nodes[1].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1773 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1774 check_added_monitors!(nodes[1], 0);
1776 let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1777 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1779 confirm_transaction(&nodes[0].block_notifier, &nodes[0].chain_monitor, &funding_tx, funding_tx.version);
1780 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1781 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1783 if restore_b_before_conf {
1784 confirm_transaction(&nodes[1].block_notifier, &nodes[1].chain_monitor, &funding_tx, funding_tx.version);
1786 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1787 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1789 for node in nodes.iter() {
1790 assert!(node.net_graph_msg_handler.handle_channel_announcement(&announcement).unwrap());
1791 node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
1792 node.net_graph_msg_handler.handle_channel_update(&bs_update).unwrap();
1795 send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1796 close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1800 fn during_funding_monitor_fail() {
1801 do_during_funding_monitor_fail(true, true);
1802 do_during_funding_monitor_fail(true, false);
1803 do_during_funding_monitor_fail(false, false);
1807 fn test_path_paused_mpp() {
1808 // Simple test of sending a multi-part payment where one path is currently blocked awaiting
1810 let chanmon_cfgs = create_chanmon_cfgs(4);
1811 let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1812 let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
1813 let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1815 let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1816 let (chan_2_ann, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known());
1817 let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1818 let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1819 let logger = test_utils::TestLogger::new();
1821 let (payment_preimage, payment_hash) = get_payment_preimage_hash!(&nodes[0]);
1822 let payment_secret = PaymentSecret([0xdb; 32]);
1823 let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler, &nodes[3].node.get_our_node_id(), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
1825 // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
1826 let path = route.paths[0].clone();
1827 route.paths.push(path);
1828 route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
1829 route.paths[0][0].short_channel_id = chan_1_id;
1830 route.paths[0][1].short_channel_id = chan_3_id;
1831 route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
1832 route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
1833 route.paths[1][1].short_channel_id = chan_4_id;
1835 // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
1836 // (for the path 0 -> 2 -> 3) fails.
1837 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1838 *nodes[0].chan_monitor.next_update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1840 // Now check that we get the right return value, indicating that the first path succeeded but
1841 // the second got a MonitorUpdateFailed err. This implies PaymentSendFailure::PartialFailure as
1842 // some paths succeeded, preventing retry.
1843 if let Err(PaymentSendFailure::PartialFailure(results)) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
1844 assert_eq!(results.len(), 2);
1845 if let Ok(()) = results[0] {} else { panic!(); }
1846 if let Err(APIError::MonitorUpdateFailed) = results[1] {} else { panic!(); }
1847 } else { panic!(); }
1848 check_added_monitors!(nodes[0], 2);
1849 *nodes[0].chan_monitor.update_ret.lock().unwrap() = Ok(());
1851 // Pass the first HTLC of the payment along to nodes[3].
1852 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1853 assert_eq!(events.len(), 1);
1854 pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 0, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), false);
1856 // And check that, after we successfully update the monitor for chan_2 we can pass the second
1857 // HTLC along to nodes[3] and claim the whole payment back to nodes[0].
1858 let (outpoint, latest_update) = nodes[0].chan_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2_id).unwrap().clone();
1859 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1860 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1861 assert_eq!(events.len(), 1);
1862 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), true);
1864 claim_payment_along_route_with_secret(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage, Some(payment_secret), 200_000);