1 // This file is Copyright its original authors, visible in version control
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
10 //! Functional tests which test the correct handling of ChannelMonitorUpdateErr returns from
12 //! There are a bunch of these as their handling is relatively error-prone so they are split out
13 //! here. See also the chanmon_fail_consistency fuzz test.
15 use chain::transaction::OutPoint;
16 use ln::channelmanager::{RAACommitmentOrder, PaymentPreimage, PaymentHash, PaymentSecret, PaymentSendFailure};
17 use ln::channelmonitor::ChannelMonitorUpdateErr;
18 use ln::features::InitFeatures;
20 use ln::msgs::{ChannelMessageHandler, ErrorAction, RoutingMessageHandler};
21 use routing::router::get_route;
22 use util::events::{Event, EventsProvider, MessageSendEvent, MessageSendEventsProvider};
23 use util::errors::APIError;
25 use bitcoin::hashes::sha256::Hash as Sha256;
26 use bitcoin::hashes::Hash;
28 use ln::functional_test_utils::*;
33 fn test_simple_monitor_permanent_update_fail() {
34 // Test that we handle a simple permanent monitor update failure
35 let chanmon_cfgs = create_chanmon_cfgs(2);
36 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
37 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
38 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
39 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
40 let logger = test_utils::TestLogger::new();
42 let (_, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
44 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::PermanentFailure);
45 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
46 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
47 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), true, APIError::ChannelUnavailable {..}, {});
48 check_added_monitors!(nodes[0], 2);
50 let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
51 assert_eq!(events_1.len(), 2);
53 MessageSendEvent::BroadcastChannelUpdate { .. } => {},
54 _ => panic!("Unexpected event"),
57 MessageSendEvent::HandleError { node_id, .. } => assert_eq!(node_id, nodes[1].node.get_our_node_id()),
58 _ => panic!("Unexpected event"),
61 // TODO: Once we hit the chain with the failure transaction we should check that we get a
62 // PaymentFailed event
64 assert_eq!(nodes[0].node.list_channels().len(), 0);
67 fn do_test_simple_monitor_temporary_update_fail(disconnect: bool) {
68 // Test that we can recover from a simple temporary monitor update failure optionally with
69 // a disconnect in between
70 let chanmon_cfgs = create_chanmon_cfgs(2);
71 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
72 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
73 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
74 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
75 let logger = test_utils::TestLogger::new();
77 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(&nodes[0]);
79 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
82 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
83 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
84 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_1, &None), false, APIError::MonitorUpdateFailed, {});
85 check_added_monitors!(nodes[0], 1);
88 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
89 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
90 assert_eq!(nodes[0].node.list_channels().len(), 1);
93 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
94 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
95 reconnect_nodes(&nodes[0], &nodes[1], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
98 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
99 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
100 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
101 check_added_monitors!(nodes[0], 0);
103 let mut events_2 = nodes[0].node.get_and_clear_pending_msg_events();
104 assert_eq!(events_2.len(), 1);
105 let payment_event = SendEvent::from_event(events_2.pop().unwrap());
106 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
107 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
108 commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
110 expect_pending_htlcs_forwardable!(nodes[1]);
112 let events_3 = nodes[1].node.get_and_clear_pending_events();
113 assert_eq!(events_3.len(), 1);
115 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
116 assert_eq!(payment_hash_1, *payment_hash);
117 assert_eq!(*payment_secret, None);
118 assert_eq!(amt, 1000000);
120 _ => panic!("Unexpected event"),
123 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
125 // Now set it to failed again...
126 let (_, payment_hash_2) = get_payment_preimage_hash!(&nodes[0]);
128 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
129 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
130 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
131 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
132 check_added_monitors!(nodes[0], 1);
135 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
136 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
137 assert_eq!(nodes[0].node.list_channels().len(), 1);
140 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
141 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
142 reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
145 // ...and make sure we can force-close a frozen channel
146 nodes[0].node.force_close_channel(&channel_id);
147 check_added_monitors!(nodes[0], 1);
148 check_closed_broadcast!(nodes[0], false);
150 // TODO: Once we hit the chain with the failure transaction we should check that we get a
151 // PaymentFailed event
153 assert_eq!(nodes[0].node.list_channels().len(), 0);
157 fn test_simple_monitor_temporary_update_fail() {
158 do_test_simple_monitor_temporary_update_fail(false);
159 do_test_simple_monitor_temporary_update_fail(true);
162 fn do_test_monitor_temporary_update_fail(disconnect_count: usize) {
163 let disconnect_flags = 8 | 16;
165 // Test that we can recover from a temporary monitor update failure with some in-flight
166 // HTLCs going on at the same time potentially with some disconnection thrown in.
167 // * First we route a payment, then get a temporary monitor update failure when trying to
168 // route a second payment. We then claim the first payment.
169 // * If disconnect_count is set, we will disconnect at this point (which is likely as
170 // TemporaryFailure likely indicates net disconnect which resulted in failing to update
171 // the ChannelMonitor on a watchtower).
172 // * If !(disconnect_count & 16) we deliver a update_fulfill_htlc/CS for the first payment
173 // immediately, otherwise we wait disconnect and deliver them via the reconnect
174 // channel_reestablish processing (ie disconnect_count & 16 makes no sense if
175 // disconnect_count & !disconnect_flags is 0).
176 // * We then update the channel monitor, reconnecting if disconnect_count is set and walk
177 // through message sending, potentially disconnect/reconnecting multiple times based on
178 // disconnect_count, to get the update_fulfill_htlc through.
179 // * We then walk through more message exchanges to get the original update_add_htlc
180 // through, swapping message ordering based on disconnect_count & 8 and optionally
181 // disconnect/reconnecting based on disconnect_count.
182 let chanmon_cfgs = create_chanmon_cfgs(2);
183 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
184 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
185 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
186 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
187 let logger = test_utils::TestLogger::new();
189 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
191 // Now try to send a second payment which will fail to send
192 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
194 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
195 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
196 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
197 unwrap_send_err!(nodes[0].node.send_payment(&route, payment_hash_2, &None), false, APIError::MonitorUpdateFailed, {});
198 check_added_monitors!(nodes[0], 1);
201 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
202 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
203 assert_eq!(nodes[0].node.list_channels().len(), 1);
205 // Claim the previous payment, which will result in a update_fulfill_htlc/CS from nodes[1]
206 // but nodes[0] won't respond since it is frozen.
207 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
208 check_added_monitors!(nodes[1], 1);
209 let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
210 assert_eq!(events_2.len(), 1);
211 let (bs_initial_fulfill, bs_initial_commitment_signed) = match events_2[0] {
212 MessageSendEvent::UpdateHTLCs { ref node_id, updates: msgs::CommitmentUpdate { ref update_add_htlcs, ref update_fulfill_htlcs, ref update_fail_htlcs, ref update_fail_malformed_htlcs, ref update_fee, ref commitment_signed } } => {
213 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
214 assert!(update_add_htlcs.is_empty());
215 assert_eq!(update_fulfill_htlcs.len(), 1);
216 assert!(update_fail_htlcs.is_empty());
217 assert!(update_fail_malformed_htlcs.is_empty());
218 assert!(update_fee.is_none());
220 if (disconnect_count & 16) == 0 {
221 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
222 let events_3 = nodes[0].node.get_and_clear_pending_events();
223 assert_eq!(events_3.len(), 1);
225 Event::PaymentSent { ref payment_preimage } => {
226 assert_eq!(*payment_preimage, payment_preimage_1);
228 _ => panic!("Unexpected event"),
231 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), commitment_signed);
232 check_added_monitors!(nodes[0], 1);
233 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
234 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
237 (update_fulfill_htlcs[0].clone(), commitment_signed.clone())
239 _ => panic!("Unexpected event"),
242 if disconnect_count & !disconnect_flags > 0 {
243 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
244 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
247 // Now fix monitor updating...
248 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
249 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
250 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
251 check_added_monitors!(nodes[0], 0);
253 macro_rules! disconnect_reconnect_peers { () => { {
254 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
255 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
257 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
258 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
259 assert_eq!(reestablish_1.len(), 1);
260 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
261 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
262 assert_eq!(reestablish_2.len(), 1);
264 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
265 let as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
266 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
267 let bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
269 assert!(as_resp.0.is_none());
270 assert!(bs_resp.0.is_none());
272 (reestablish_1, reestablish_2, as_resp, bs_resp)
275 let (payment_event, initial_revoke_and_ack) = if disconnect_count & !disconnect_flags > 0 {
276 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
277 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
279 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
280 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
281 assert_eq!(reestablish_1.len(), 1);
282 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
283 let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
284 assert_eq!(reestablish_2.len(), 1);
286 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
287 check_added_monitors!(nodes[0], 0);
288 let mut as_resp = handle_chan_reestablish_msgs!(nodes[0], nodes[1]);
289 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
290 check_added_monitors!(nodes[1], 0);
291 let mut bs_resp = handle_chan_reestablish_msgs!(nodes[1], nodes[0]);
293 assert!(as_resp.0.is_none());
294 assert!(bs_resp.0.is_none());
296 assert!(bs_resp.1.is_none());
297 if (disconnect_count & 16) == 0 {
298 assert!(bs_resp.2.is_none());
300 assert!(as_resp.1.is_some());
301 assert!(as_resp.2.is_some());
302 assert!(as_resp.3 == RAACommitmentOrder::CommitmentFirst);
304 assert!(bs_resp.2.as_ref().unwrap().update_add_htlcs.is_empty());
305 assert!(bs_resp.2.as_ref().unwrap().update_fail_htlcs.is_empty());
306 assert!(bs_resp.2.as_ref().unwrap().update_fail_malformed_htlcs.is_empty());
307 assert!(bs_resp.2.as_ref().unwrap().update_fee.is_none());
308 assert!(bs_resp.2.as_ref().unwrap().update_fulfill_htlcs == vec![bs_initial_fulfill]);
309 assert!(bs_resp.2.as_ref().unwrap().commitment_signed == bs_initial_commitment_signed);
311 assert!(as_resp.1.is_none());
313 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().update_fulfill_htlcs[0]);
314 let events_3 = nodes[0].node.get_and_clear_pending_events();
315 assert_eq!(events_3.len(), 1);
317 Event::PaymentSent { ref payment_preimage } => {
318 assert_eq!(*payment_preimage, payment_preimage_1);
320 _ => panic!("Unexpected event"),
323 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_resp.2.as_ref().unwrap().commitment_signed);
324 let as_resp_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
325 // No commitment_signed so get_event_msg's assert(len == 1) passes
326 check_added_monitors!(nodes[0], 1);
328 as_resp.1 = Some(as_resp_raa);
332 if disconnect_count & !disconnect_flags > 1 {
333 let (second_reestablish_1, second_reestablish_2, second_as_resp, second_bs_resp) = disconnect_reconnect_peers!();
335 if (disconnect_count & 16) == 0 {
336 assert!(reestablish_1 == second_reestablish_1);
337 assert!(reestablish_2 == second_reestablish_2);
339 assert!(as_resp == second_as_resp);
340 assert!(bs_resp == second_bs_resp);
343 (SendEvent::from_commitment_update(nodes[1].node.get_our_node_id(), as_resp.2.unwrap()), as_resp.1.unwrap())
345 let mut events_4 = nodes[0].node.get_and_clear_pending_msg_events();
346 assert_eq!(events_4.len(), 2);
347 (SendEvent::from_event(events_4.remove(0)), match events_4[0] {
348 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
349 assert_eq!(*node_id, nodes[1].node.get_our_node_id());
352 _ => panic!("Unexpected event"),
356 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
358 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
359 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
360 let bs_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
361 // nodes[1] is awaiting an RAA from nodes[0] still so get_event_msg's assert(len == 1) passes
362 check_added_monitors!(nodes[1], 1);
364 if disconnect_count & !disconnect_flags > 2 {
365 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
367 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
368 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
370 assert!(as_resp.2.is_none());
371 assert!(bs_resp.2.is_none());
374 let as_commitment_update;
375 let bs_second_commitment_update;
377 macro_rules! handle_bs_raa { () => {
378 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_revoke_and_ack);
379 as_commitment_update = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
380 assert!(as_commitment_update.update_add_htlcs.is_empty());
381 assert!(as_commitment_update.update_fulfill_htlcs.is_empty());
382 assert!(as_commitment_update.update_fail_htlcs.is_empty());
383 assert!(as_commitment_update.update_fail_malformed_htlcs.is_empty());
384 assert!(as_commitment_update.update_fee.is_none());
385 check_added_monitors!(nodes[0], 1);
388 macro_rules! handle_initial_raa { () => {
389 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &initial_revoke_and_ack);
390 bs_second_commitment_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
391 assert!(bs_second_commitment_update.update_add_htlcs.is_empty());
392 assert!(bs_second_commitment_update.update_fulfill_htlcs.is_empty());
393 assert!(bs_second_commitment_update.update_fail_htlcs.is_empty());
394 assert!(bs_second_commitment_update.update_fail_malformed_htlcs.is_empty());
395 assert!(bs_second_commitment_update.update_fee.is_none());
396 check_added_monitors!(nodes[1], 1);
399 if (disconnect_count & 8) == 0 {
402 if disconnect_count & !disconnect_flags > 3 {
403 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
405 assert!(as_resp.1.unwrap() == initial_revoke_and_ack);
406 assert!(bs_resp.1.is_none());
408 assert!(as_resp.2.unwrap() == as_commitment_update);
409 assert!(bs_resp.2.is_none());
411 assert!(as_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
414 handle_initial_raa!();
416 if disconnect_count & !disconnect_flags > 4 {
417 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
419 assert!(as_resp.1.is_none());
420 assert!(bs_resp.1.is_none());
422 assert!(as_resp.2.unwrap() == as_commitment_update);
423 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
426 handle_initial_raa!();
428 if disconnect_count & !disconnect_flags > 3 {
429 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
431 assert!(as_resp.1.is_none());
432 assert!(bs_resp.1.unwrap() == bs_revoke_and_ack);
434 assert!(as_resp.2.is_none());
435 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
437 assert!(bs_resp.3 == RAACommitmentOrder::RevokeAndACKFirst);
442 if disconnect_count & !disconnect_flags > 4 {
443 let (_, _, as_resp, bs_resp) = disconnect_reconnect_peers!();
445 assert!(as_resp.1.is_none());
446 assert!(bs_resp.1.is_none());
448 assert!(as_resp.2.unwrap() == as_commitment_update);
449 assert!(bs_resp.2.unwrap() == bs_second_commitment_update);
453 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_second_commitment_update.commitment_signed);
454 let as_revoke_and_ack = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
455 // No commitment_signed so get_event_msg's assert(len == 1) passes
456 check_added_monitors!(nodes[0], 1);
458 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment_update.commitment_signed);
459 let bs_second_revoke_and_ack = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
460 // No commitment_signed so get_event_msg's assert(len == 1) passes
461 check_added_monitors!(nodes[1], 1);
463 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_revoke_and_ack);
464 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
465 check_added_monitors!(nodes[1], 1);
467 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_revoke_and_ack);
468 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
469 check_added_monitors!(nodes[0], 1);
471 expect_pending_htlcs_forwardable!(nodes[1]);
473 let events_5 = nodes[1].node.get_and_clear_pending_events();
474 assert_eq!(events_5.len(), 1);
476 Event::PaymentReceived { ref payment_hash, ref payment_secret, amt } => {
477 assert_eq!(payment_hash_2, *payment_hash);
478 assert_eq!(*payment_secret, None);
479 assert_eq!(amt, 1000000);
481 _ => panic!("Unexpected event"),
484 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
488 fn test_monitor_temporary_update_fail_a() {
489 do_test_monitor_temporary_update_fail(0);
490 do_test_monitor_temporary_update_fail(1);
491 do_test_monitor_temporary_update_fail(2);
492 do_test_monitor_temporary_update_fail(3);
493 do_test_monitor_temporary_update_fail(4);
494 do_test_monitor_temporary_update_fail(5);
498 fn test_monitor_temporary_update_fail_b() {
499 do_test_monitor_temporary_update_fail(2 | 8);
500 do_test_monitor_temporary_update_fail(3 | 8);
501 do_test_monitor_temporary_update_fail(4 | 8);
502 do_test_monitor_temporary_update_fail(5 | 8);
506 fn test_monitor_temporary_update_fail_c() {
507 do_test_monitor_temporary_update_fail(1 | 16);
508 do_test_monitor_temporary_update_fail(2 | 16);
509 do_test_monitor_temporary_update_fail(3 | 16);
510 do_test_monitor_temporary_update_fail(2 | 8 | 16);
511 do_test_monitor_temporary_update_fail(3 | 8 | 16);
515 fn test_monitor_update_fail_cs() {
516 // Tests handling of a monitor update failure when processing an incoming commitment_signed
517 let chanmon_cfgs = create_chanmon_cfgs(2);
518 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
519 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
520 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
521 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
522 let logger = test_utils::TestLogger::new();
524 let (payment_preimage, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
526 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
527 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
528 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
529 check_added_monitors!(nodes[0], 1);
532 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
533 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
535 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
536 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event.commitment_msg);
537 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
538 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
539 check_added_monitors!(nodes[1], 1);
540 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
542 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
543 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
544 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
545 check_added_monitors!(nodes[1], 0);
546 let responses = nodes[1].node.get_and_clear_pending_msg_events();
547 assert_eq!(responses.len(), 2);
550 MessageSendEvent::SendRevokeAndACK { ref msg, ref node_id } => {
551 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
552 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &msg);
553 check_added_monitors!(nodes[0], 1);
555 _ => panic!("Unexpected event"),
558 MessageSendEvent::UpdateHTLCs { ref updates, ref node_id } => {
559 assert!(updates.update_add_htlcs.is_empty());
560 assert!(updates.update_fulfill_htlcs.is_empty());
561 assert!(updates.update_fail_htlcs.is_empty());
562 assert!(updates.update_fail_malformed_htlcs.is_empty());
563 assert!(updates.update_fee.is_none());
564 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
566 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
567 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
568 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
569 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
570 check_added_monitors!(nodes[0], 1);
571 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
573 _ => panic!("Unexpected event"),
576 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
577 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
578 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
579 check_added_monitors!(nodes[0], 0);
581 let final_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
582 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &final_raa);
583 check_added_monitors!(nodes[1], 1);
585 expect_pending_htlcs_forwardable!(nodes[1]);
587 let events = nodes[1].node.get_and_clear_pending_events();
588 assert_eq!(events.len(), 1);
590 Event::PaymentReceived { payment_hash, payment_secret, amt } => {
591 assert_eq!(payment_hash, our_payment_hash);
592 assert_eq!(payment_secret, None);
593 assert_eq!(amt, 1000000);
595 _ => panic!("Unexpected event"),
598 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage, 1_000_000);
602 fn test_monitor_update_fail_no_rebroadcast() {
603 // Tests handling of a monitor update failure when no message rebroadcasting on
604 // channel_monitor_updated() is required. Backported from chanmon_fail_consistency
606 let chanmon_cfgs = create_chanmon_cfgs(2);
607 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
608 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
609 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
610 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
611 let logger = test_utils::TestLogger::new();
613 let (payment_preimage_1, our_payment_hash) = get_payment_preimage_hash!(nodes[0]);
615 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
616 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
617 nodes[0].node.send_payment(&route, our_payment_hash, &None).unwrap();
618 check_added_monitors!(nodes[0], 1);
621 let send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
622 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
623 let bs_raa = commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true, false, true);
625 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
626 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &bs_raa);
627 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
628 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
629 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
630 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
631 check_added_monitors!(nodes[1], 1);
633 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
634 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
635 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
636 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
637 check_added_monitors!(nodes[1], 0);
638 expect_pending_htlcs_forwardable!(nodes[1]);
640 let events = nodes[1].node.get_and_clear_pending_events();
641 assert_eq!(events.len(), 1);
643 Event::PaymentReceived { payment_hash, .. } => {
644 assert_eq!(payment_hash, our_payment_hash);
646 _ => panic!("Unexpected event"),
649 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
653 fn test_monitor_update_raa_while_paused() {
654 // Tests handling of an RAA while monitor updating has already been marked failed.
655 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
656 let chanmon_cfgs = create_chanmon_cfgs(2);
657 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
658 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
659 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
660 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
661 let logger = test_utils::TestLogger::new();
663 send_payment(&nodes[0], &[&nodes[1]], 5000000, 5_000_000);
664 let (payment_preimage_1, our_payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
666 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
667 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
668 nodes[0].node.send_payment(&route, our_payment_hash_1, &None).unwrap();
669 check_added_monitors!(nodes[0], 1);
671 let send_event_1 = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
673 let (payment_preimage_2, our_payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
675 let net_graph_msg_handler = &nodes[1].net_graph_msg_handler;
676 let route = get_route(&nodes[1].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
677 nodes[1].node.send_payment(&route, our_payment_hash_2, &None).unwrap();
678 check_added_monitors!(nodes[1], 1);
680 let send_event_2 = SendEvent::from_event(nodes[1].node.get_and_clear_pending_msg_events().remove(0));
682 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event_1.msgs[0]);
683 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &send_event_1.commitment_msg);
684 check_added_monitors!(nodes[1], 1);
685 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
687 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
688 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_2.msgs[0]);
689 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_2.commitment_msg);
690 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
691 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
692 check_added_monitors!(nodes[0], 1);
694 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
695 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
696 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
697 check_added_monitors!(nodes[0], 1);
699 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
700 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
701 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
702 check_added_monitors!(nodes[0], 0);
704 let as_update_raa = get_revoke_commit_msgs!(nodes[0], nodes[1].node.get_our_node_id());
705 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_update_raa.0);
706 check_added_monitors!(nodes[1], 1);
707 let bs_cs = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
709 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_update_raa.1);
710 check_added_monitors!(nodes[1], 1);
711 let bs_second_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
713 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_cs.commitment_signed);
714 check_added_monitors!(nodes[0], 1);
715 let as_second_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
717 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_second_raa);
718 check_added_monitors!(nodes[0], 1);
719 expect_pending_htlcs_forwardable!(nodes[0]);
720 expect_payment_received!(nodes[0], our_payment_hash_2, 1000000);
722 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_second_raa);
723 check_added_monitors!(nodes[1], 1);
724 expect_pending_htlcs_forwardable!(nodes[1]);
725 expect_payment_received!(nodes[1], our_payment_hash_1, 1000000);
727 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
728 claim_payment(&nodes[1], &[&nodes[0]], payment_preimage_2, 1_000_000);
731 fn do_test_monitor_update_fail_raa(test_ignore_second_cs: bool) {
732 // Tests handling of a monitor update failure when processing an incoming RAA
733 let chanmon_cfgs = create_chanmon_cfgs(3);
734 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
735 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
736 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
737 create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
738 let chan_2 = create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
739 let logger = test_utils::TestLogger::new();
741 // Rebalance a bit so that we can send backwards from 2 to 1.
742 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
744 // Route a first payment that we'll fail backwards
745 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
747 // Fail the payment backwards, failing the monitor update on nodes[1]'s receipt of the RAA
748 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
749 expect_pending_htlcs_forwardable!(nodes[2]);
750 check_added_monitors!(nodes[2], 1);
752 let updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
753 assert!(updates.update_add_htlcs.is_empty());
754 assert!(updates.update_fulfill_htlcs.is_empty());
755 assert_eq!(updates.update_fail_htlcs.len(), 1);
756 assert!(updates.update_fail_malformed_htlcs.is_empty());
757 assert!(updates.update_fee.is_none());
758 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
760 let bs_revoke_and_ack = commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false, true, false, true);
761 check_added_monitors!(nodes[0], 0);
763 // While the second channel is AwaitingRAA, forward a second payment to get it into the
765 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
767 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
768 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
769 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
770 check_added_monitors!(nodes[0], 1);
773 let mut send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
774 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
775 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false);
777 expect_pending_htlcs_forwardable!(nodes[1]);
778 check_added_monitors!(nodes[1], 0);
779 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
781 // Now fail monitor updating.
782 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
783 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
784 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
785 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
786 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
787 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
788 check_added_monitors!(nodes[1], 1);
790 // Attempt to forward a third payment but fail due to the second channel being unavailable
792 let (_, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
794 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
795 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[2].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
796 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
797 check_added_monitors!(nodes[0], 1);
800 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(()); // We succeed in updating the monitor for the first channel
801 send_event = SendEvent::from_event(nodes[0].node.get_and_clear_pending_msg_events().remove(0));
802 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &send_event.msgs[0]);
803 commitment_signed_dance!(nodes[1], nodes[0], send_event.commitment_msg, false, true);
804 check_added_monitors!(nodes[1], 0);
806 let mut events_2 = nodes[1].node.get_and_clear_pending_msg_events();
807 assert_eq!(events_2.len(), 1);
808 match events_2.remove(0) {
809 MessageSendEvent::UpdateHTLCs { node_id, updates } => {
810 assert_eq!(node_id, nodes[0].node.get_our_node_id());
811 assert!(updates.update_fulfill_htlcs.is_empty());
812 assert_eq!(updates.update_fail_htlcs.len(), 1);
813 assert!(updates.update_fail_malformed_htlcs.is_empty());
814 assert!(updates.update_add_htlcs.is_empty());
815 assert!(updates.update_fee.is_none());
817 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fail_htlcs[0]);
818 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false, true);
820 let msg_events = nodes[0].node.get_and_clear_pending_msg_events();
821 assert_eq!(msg_events.len(), 1);
822 match msg_events[0] {
823 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
824 assert_eq!(msg.contents.short_channel_id, chan_2.0.contents.short_channel_id);
825 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
827 _ => panic!("Unexpected event"),
830 let events = nodes[0].node.get_and_clear_pending_events();
831 assert_eq!(events.len(), 1);
832 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
833 assert_eq!(payment_hash, payment_hash_3);
834 assert!(!rejected_by_dest);
835 } else { panic!("Unexpected event!"); }
837 _ => panic!("Unexpected event type!"),
840 let (payment_preimage_4, payment_hash_4) = if test_ignore_second_cs {
841 // Try to route another payment backwards from 2 to make sure 1 holds off on responding
842 let (payment_preimage_4, payment_hash_4) = get_payment_preimage_hash!(nodes[0]);
843 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
844 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
845 nodes[2].node.send_payment(&route, payment_hash_4, &None).unwrap();
846 check_added_monitors!(nodes[2], 1);
848 send_event = SendEvent::from_event(nodes[2].node.get_and_clear_pending_msg_events().remove(0));
849 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &send_event.msgs[0]);
850 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &send_event.commitment_msg);
851 check_added_monitors!(nodes[1], 1);
852 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
853 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
854 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
855 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
856 (Some(payment_preimage_4), Some(payment_hash_4))
857 } else { (None, None) };
859 // Restore monitor updating, ensuring we immediately get a fail-back update and a
860 // update_add update.
861 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
862 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2.2).unwrap().clone();
863 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
864 check_added_monitors!(nodes[1], 0);
865 expect_pending_htlcs_forwardable!(nodes[1]);
866 check_added_monitors!(nodes[1], 1);
868 let mut events_3 = nodes[1].node.get_and_clear_pending_msg_events();
869 if test_ignore_second_cs {
870 assert_eq!(events_3.len(), 3);
872 assert_eq!(events_3.len(), 2);
875 // Note that the ordering of the events for different nodes is non-prescriptive, though the
876 // ordering of the two events that both go to nodes[2] have to stay in the same order.
877 let messages_a = match events_3.pop().unwrap() {
878 MessageSendEvent::UpdateHTLCs { node_id, mut updates } => {
879 assert_eq!(node_id, nodes[0].node.get_our_node_id());
880 assert!(updates.update_fulfill_htlcs.is_empty());
881 assert_eq!(updates.update_fail_htlcs.len(), 1);
882 assert!(updates.update_fail_malformed_htlcs.is_empty());
883 assert!(updates.update_add_htlcs.is_empty());
884 assert!(updates.update_fee.is_none());
885 (updates.update_fail_htlcs.remove(0), updates.commitment_signed)
887 _ => panic!("Unexpected event type!"),
889 let raa = if test_ignore_second_cs {
890 match events_3.remove(1) {
891 MessageSendEvent::SendRevokeAndACK { node_id, msg } => {
892 assert_eq!(node_id, nodes[2].node.get_our_node_id());
895 _ => panic!("Unexpected event"),
898 let send_event_b = SendEvent::from_event(events_3.remove(0));
899 assert_eq!(send_event_b.node_id, nodes[2].node.get_our_node_id());
901 // Now deliver the new messages...
903 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &messages_a.0);
904 commitment_signed_dance!(nodes[0], nodes[1], messages_a.1, false);
905 let events_4 = nodes[0].node.get_and_clear_pending_events();
906 assert_eq!(events_4.len(), 1);
907 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events_4[0] {
908 assert_eq!(payment_hash, payment_hash_1);
909 assert!(rejected_by_dest);
910 } else { panic!("Unexpected event!"); }
912 nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event_b.msgs[0]);
913 if test_ignore_second_cs {
914 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &send_event_b.commitment_msg);
915 check_added_monitors!(nodes[2], 1);
916 let bs_revoke_and_ack = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
917 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa.unwrap());
918 check_added_monitors!(nodes[2], 1);
919 let bs_cs = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
920 assert!(bs_cs.update_add_htlcs.is_empty());
921 assert!(bs_cs.update_fail_htlcs.is_empty());
922 assert!(bs_cs.update_fail_malformed_htlcs.is_empty());
923 assert!(bs_cs.update_fulfill_htlcs.is_empty());
924 assert!(bs_cs.update_fee.is_none());
926 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_revoke_and_ack);
927 check_added_monitors!(nodes[1], 1);
928 let as_cs = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
929 assert!(as_cs.update_add_htlcs.is_empty());
930 assert!(as_cs.update_fail_htlcs.is_empty());
931 assert!(as_cs.update_fail_malformed_htlcs.is_empty());
932 assert!(as_cs.update_fulfill_htlcs.is_empty());
933 assert!(as_cs.update_fee.is_none());
935 nodes[1].node.handle_commitment_signed(&nodes[2].node.get_our_node_id(), &bs_cs.commitment_signed);
936 check_added_monitors!(nodes[1], 1);
937 let as_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[2].node.get_our_node_id());
939 nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &as_cs.commitment_signed);
940 check_added_monitors!(nodes[2], 1);
941 let bs_second_raa = get_event_msg!(nodes[2], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
943 nodes[2].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &as_raa);
944 check_added_monitors!(nodes[2], 1);
945 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
947 nodes[1].node.handle_revoke_and_ack(&nodes[2].node.get_our_node_id(), &bs_second_raa);
948 check_added_monitors!(nodes[1], 1);
949 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
951 commitment_signed_dance!(nodes[2], nodes[1], send_event_b.commitment_msg, false);
954 expect_pending_htlcs_forwardable!(nodes[2]);
956 let events_6 = nodes[2].node.get_and_clear_pending_events();
957 assert_eq!(events_6.len(), 1);
959 Event::PaymentReceived { payment_hash, .. } => { assert_eq!(payment_hash, payment_hash_2); },
960 _ => panic!("Unexpected event"),
963 if test_ignore_second_cs {
964 expect_pending_htlcs_forwardable!(nodes[1]);
965 check_added_monitors!(nodes[1], 1);
967 send_event = SendEvent::from_node(&nodes[1]);
968 assert_eq!(send_event.node_id, nodes[0].node.get_our_node_id());
969 assert_eq!(send_event.msgs.len(), 1);
970 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &send_event.msgs[0]);
971 commitment_signed_dance!(nodes[0], nodes[1], send_event.commitment_msg, false);
973 expect_pending_htlcs_forwardable!(nodes[0]);
975 let events_9 = nodes[0].node.get_and_clear_pending_events();
976 assert_eq!(events_9.len(), 1);
978 Event::PaymentReceived { payment_hash, .. } => assert_eq!(payment_hash, payment_hash_4.unwrap()),
979 _ => panic!("Unexpected event"),
981 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_4.unwrap(), 1_000_000);
984 claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2, 1_000_000);
988 fn test_monitor_update_fail_raa() {
989 do_test_monitor_update_fail_raa(false);
990 do_test_monitor_update_fail_raa(true);
994 fn test_monitor_update_fail_reestablish() {
995 // Simple test for message retransmission after monitor update failure on
996 // channel_reestablish generating a monitor update (which comes from freeing holding cell
998 let chanmon_cfgs = create_chanmon_cfgs(3);
999 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1000 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1001 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1002 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1003 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1005 let (our_payment_preimage, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1007 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1008 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1010 assert!(nodes[2].node.claim_funds(our_payment_preimage, &None, 1_000_000));
1011 check_added_monitors!(nodes[2], 1);
1012 let mut updates = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1013 assert!(updates.update_add_htlcs.is_empty());
1014 assert!(updates.update_fail_htlcs.is_empty());
1015 assert!(updates.update_fail_malformed_htlcs.is_empty());
1016 assert!(updates.update_fee.is_none());
1017 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1018 nodes[1].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1019 check_added_monitors!(nodes[1], 1);
1020 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1021 commitment_signed_dance!(nodes[1], nodes[2], updates.commitment_signed, false);
1023 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1024 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1025 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1027 let as_reestablish = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1028 let bs_reestablish = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1030 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1032 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1033 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1034 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1035 check_added_monitors!(nodes[1], 1);
1037 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1038 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1040 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1041 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1043 assert!(as_reestablish == get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id()));
1044 assert!(bs_reestablish == get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id()));
1046 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
1048 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
1049 check_added_monitors!(nodes[1], 0);
1050 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1052 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1053 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1054 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1055 check_added_monitors!(nodes[1], 0);
1057 updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1058 assert!(updates.update_add_htlcs.is_empty());
1059 assert!(updates.update_fail_htlcs.is_empty());
1060 assert!(updates.update_fail_malformed_htlcs.is_empty());
1061 assert!(updates.update_fee.is_none());
1062 assert_eq!(updates.update_fulfill_htlcs.len(), 1);
1063 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1064 commitment_signed_dance!(nodes[0], nodes[1], updates.commitment_signed, false);
1066 let events = nodes[0].node.get_and_clear_pending_events();
1067 assert_eq!(events.len(), 1);
1069 Event::PaymentSent { payment_preimage, .. } => assert_eq!(payment_preimage, our_payment_preimage),
1070 _ => panic!("Unexpected event"),
1075 fn raa_no_response_awaiting_raa_state() {
1076 // This is a rather convoluted test which ensures that if handling of an RAA does not happen
1077 // due to a previous monitor update failure, we still set AwaitingRemoteRevoke on the channel
1078 // in question (assuming it intends to respond with a CS after monitor updating is restored).
1079 // Backported from chanmon_fail_consistency fuzz tests as this used to be broken.
1080 let chanmon_cfgs = create_chanmon_cfgs(2);
1081 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1082 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1083 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1084 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1085 let logger = test_utils::TestLogger::new();
1087 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1088 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1089 let (payment_preimage_3, payment_hash_3) = get_payment_preimage_hash!(nodes[0]);
1091 // Queue up two payments - one will be delivered right away, one immediately goes into the
1092 // holding cell as nodes[0] is AwaitingRAA. Ultimately this allows us to deliver an RAA
1093 // immediately after a CS. By setting failing the monitor update failure from the CS (which
1094 // requires only an RAA response due to AwaitingRAA) we can deliver the RAA and require the CS
1095 // generation during RAA while in monitor-update-failed state.
1097 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1098 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1099 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1100 check_added_monitors!(nodes[0], 1);
1101 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1102 check_added_monitors!(nodes[0], 0);
1105 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1106 assert_eq!(events.len(), 1);
1107 let payment_event = SendEvent::from_event(events.pop().unwrap());
1108 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1109 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1110 check_added_monitors!(nodes[1], 1);
1112 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1113 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1114 check_added_monitors!(nodes[0], 1);
1115 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1116 assert_eq!(events.len(), 1);
1117 let payment_event = SendEvent::from_event(events.pop().unwrap());
1119 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1120 check_added_monitors!(nodes[0], 1);
1121 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1123 // Now we have a CS queued up which adds a new HTLC (which will need a RAA/CS response from
1124 // nodes[1]) followed by an RAA. Fail the monitor updating prior to the CS, deliver the RAA,
1125 // then restore channel monitor updates.
1126 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1127 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1128 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1129 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1130 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1131 check_added_monitors!(nodes[1], 1);
1133 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1134 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1135 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented responses to RAA".to_string(), 1);
1136 check_added_monitors!(nodes[1], 1);
1138 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1139 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1140 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1141 // nodes[1] should be AwaitingRAA here!
1142 check_added_monitors!(nodes[1], 0);
1143 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1144 expect_pending_htlcs_forwardable!(nodes[1]);
1145 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1147 // We send a third payment here, which is somewhat of a redundant test, but the
1148 // chanmon_fail_consistency test required it to actually find the bug (by seeing out-of-sync
1149 // commitment transaction states) whereas here we can explicitly check for it.
1151 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1152 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1153 nodes[0].node.send_payment(&route, payment_hash_3, &None).unwrap();
1154 check_added_monitors!(nodes[0], 0);
1155 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1157 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1158 check_added_monitors!(nodes[0], 1);
1159 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1160 assert_eq!(events.len(), 1);
1161 let payment_event = SendEvent::from_event(events.pop().unwrap());
1163 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1164 check_added_monitors!(nodes[0], 1);
1165 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1167 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1168 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1169 check_added_monitors!(nodes[1], 1);
1170 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1172 // Finally deliver the RAA to nodes[1] which results in a CS response to the last update
1173 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1174 check_added_monitors!(nodes[1], 1);
1175 expect_pending_htlcs_forwardable!(nodes[1]);
1176 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1177 let bs_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1179 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1180 check_added_monitors!(nodes[0], 1);
1182 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_update.commitment_signed);
1183 check_added_monitors!(nodes[0], 1);
1184 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1186 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1187 check_added_monitors!(nodes[1], 1);
1188 expect_pending_htlcs_forwardable!(nodes[1]);
1189 expect_payment_received!(nodes[1], payment_hash_3, 1000000);
1191 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1192 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1193 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_3, 1_000_000);
1197 fn claim_while_disconnected_monitor_update_fail() {
1198 // Test for claiming a payment while disconnected and then having the resulting
1199 // channel-update-generated monitor update fail. This kind of thing isn't a particularly
1200 // contrived case for nodes with network instability.
1201 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1202 // code introduced a regression in this test (specifically, this caught a removal of the
1203 // channel_reestablish handling ensuring the order was sensical given the messages used).
1204 let chanmon_cfgs = create_chanmon_cfgs(2);
1205 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1206 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1207 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1208 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1209 let logger = test_utils::TestLogger::new();
1211 // Forward a payment for B to claim
1212 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1214 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1215 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1217 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1218 check_added_monitors!(nodes[1], 1);
1220 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1221 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1223 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1224 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1226 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1227 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1229 // Now deliver a's reestablish, freeing the claim from the holding cell, but fail the monitor
1231 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1233 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1234 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1235 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1236 check_added_monitors!(nodes[1], 1);
1237 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1239 // Send a second payment from A to B, resulting in a commitment update that gets swallowed with
1240 // the monitor still failed
1241 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1243 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1244 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1245 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1246 check_added_monitors!(nodes[0], 1);
1249 let as_updates = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1250 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &as_updates.update_add_htlcs[0]);
1251 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_updates.commitment_signed);
1252 check_added_monitors!(nodes[1], 1);
1253 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1254 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1255 // Note that nodes[1] not updating monitor here is OK - it wont take action on the new HTLC
1256 // until we've channel_monitor_update'd and updated for the new commitment transaction.
1258 // Now un-fail the monitor, which will result in B sending its original commitment update,
1259 // receiving the commitment update from A, and the resulting commitment dances.
1260 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1261 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1262 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1263 check_added_monitors!(nodes[1], 0);
1265 let bs_msgs = nodes[1].node.get_and_clear_pending_msg_events();
1266 assert_eq!(bs_msgs.len(), 2);
1269 MessageSendEvent::UpdateHTLCs { ref node_id, ref updates } => {
1270 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1271 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
1272 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &updates.commitment_signed);
1273 check_added_monitors!(nodes[0], 1);
1275 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1276 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1277 check_added_monitors!(nodes[1], 1);
1279 _ => panic!("Unexpected event"),
1283 MessageSendEvent::SendRevokeAndACK { ref node_id, ref msg } => {
1284 assert_eq!(*node_id, nodes[0].node.get_our_node_id());
1285 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), msg);
1286 check_added_monitors!(nodes[0], 1);
1288 _ => panic!("Unexpected event"),
1291 let as_commitment = get_htlc_update_msgs!(nodes[0], nodes[1].node.get_our_node_id());
1293 let bs_commitment = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1294 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_commitment.commitment_signed);
1295 check_added_monitors!(nodes[0], 1);
1296 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1298 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &as_commitment.commitment_signed);
1299 check_added_monitors!(nodes[1], 1);
1300 let bs_raa = get_event_msg!(nodes[1], MessageSendEvent::SendRevokeAndACK, nodes[0].node.get_our_node_id());
1301 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1302 check_added_monitors!(nodes[1], 1);
1304 expect_pending_htlcs_forwardable!(nodes[1]);
1305 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1307 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_raa);
1308 check_added_monitors!(nodes[0], 1);
1310 let events = nodes[0].node.get_and_clear_pending_events();
1311 assert_eq!(events.len(), 1);
1313 Event::PaymentSent { ref payment_preimage } => {
1314 assert_eq!(*payment_preimage, payment_preimage_1);
1316 _ => panic!("Unexpected event"),
1319 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1323 fn monitor_failed_no_reestablish_response() {
1324 // Test for receiving a channel_reestablish after a monitor update failure resulted in no
1325 // response to a commitment_signed.
1326 // Backported from chanmon_fail_consistency fuzz tests as it caught a long-standing
1327 // debug_assert!() failure in channel_reestablish handling.
1328 let chanmon_cfgs = create_chanmon_cfgs(2);
1329 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1330 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1331 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1332 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1333 let logger = test_utils::TestLogger::new();
1335 // Route the payment and deliver the initial commitment_signed (with a monitor update failure
1337 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1339 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1340 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1341 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1342 check_added_monitors!(nodes[0], 1);
1345 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1346 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1347 assert_eq!(events.len(), 1);
1348 let payment_event = SendEvent::from_event(events.pop().unwrap());
1349 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1350 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1351 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1352 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1353 check_added_monitors!(nodes[1], 1);
1355 // Now disconnect and immediately reconnect, delivering the channel_reestablish while nodes[1]
1356 // is still failing to update monitors.
1357 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1358 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1360 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1361 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: InitFeatures::empty() });
1363 let as_reconnect = get_event_msg!(nodes[0], MessageSendEvent::SendChannelReestablish, nodes[1].node.get_our_node_id());
1364 let bs_reconnect = get_event_msg!(nodes[1], MessageSendEvent::SendChannelReestablish, nodes[0].node.get_our_node_id());
1366 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reconnect);
1367 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reconnect);
1369 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1370 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1371 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1372 check_added_monitors!(nodes[1], 0);
1373 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1375 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1376 check_added_monitors!(nodes[0], 1);
1377 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1378 check_added_monitors!(nodes[0], 1);
1380 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1381 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1382 check_added_monitors!(nodes[1], 1);
1384 expect_pending_htlcs_forwardable!(nodes[1]);
1385 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1387 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1391 fn first_message_on_recv_ordering() {
1392 // Test that if the initial generator of a monitor-update-frozen state doesn't generate
1393 // messages, we're willing to flip the order of response messages if neccessary in resposne to
1394 // a commitment_signed which needs to send an RAA first.
1395 // At a high level, our goal is to fail monitor updating in response to an RAA which needs no
1396 // response and then handle a CS while in the failed state, requiring an RAA followed by a CS
1397 // response. To do this, we start routing two payments, with the final RAA for the first being
1398 // delivered while B is in AwaitingRAA, hence when we deliver the CS for the second B will
1399 // have no pending response but will want to send a RAA/CS (with the updates for the second
1400 // payment applied).
1401 // Backported from chanmon_fail_consistency fuzz tests as it caught a bug here.
1402 let chanmon_cfgs = create_chanmon_cfgs(2);
1403 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1404 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1405 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1406 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1407 let logger = test_utils::TestLogger::new();
1409 // Route the first payment outbound, holding the last RAA for B until we are set up so that we
1410 // can deliver it and fail the monitor update.
1411 let (payment_preimage_1, payment_hash_1) = get_payment_preimage_hash!(nodes[0]);
1413 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1414 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1415 nodes[0].node.send_payment(&route, payment_hash_1, &None).unwrap();
1416 check_added_monitors!(nodes[0], 1);
1419 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1420 assert_eq!(events.len(), 1);
1421 let payment_event = SendEvent::from_event(events.pop().unwrap());
1422 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1423 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1424 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1425 check_added_monitors!(nodes[1], 1);
1426 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1428 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1429 check_added_monitors!(nodes[0], 1);
1430 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1431 check_added_monitors!(nodes[0], 1);
1433 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1435 // Route the second payment, generating an update_add_htlc/commitment_signed
1436 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1438 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1439 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1440 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1441 check_added_monitors!(nodes[0], 1);
1443 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1444 assert_eq!(events.len(), 1);
1445 let payment_event = SendEvent::from_event(events.pop().unwrap());
1446 assert_eq!(payment_event.node_id, nodes[1].node.get_our_node_id());
1448 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1450 // Deliver the final RAA for the first payment, which does not require a response. RAAs
1451 // generally require a commitment_signed, so the fact that we're expecting an opposite response
1452 // to the next message also tests resetting the delivery order.
1453 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1454 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1455 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1456 check_added_monitors!(nodes[1], 1);
1458 // Now deliver the update_add_htlc/commitment_signed for the second payment, which does need an
1459 // RAA/CS response, which should be generated when we call channel_monitor_update (with the
1460 // appropriate HTLC acceptance).
1461 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1462 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &payment_event.commitment_msg);
1463 check_added_monitors!(nodes[1], 1);
1464 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1465 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Previous monitor update failure prevented generation of RAA".to_string(), 1);
1467 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1468 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1469 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1470 check_added_monitors!(nodes[1], 0);
1472 expect_pending_htlcs_forwardable!(nodes[1]);
1473 expect_payment_received!(nodes[1], payment_hash_1, 1000000);
1475 let bs_responses = get_revoke_commit_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1476 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &bs_responses.0);
1477 check_added_monitors!(nodes[0], 1);
1478 nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &bs_responses.1);
1479 check_added_monitors!(nodes[0], 1);
1481 let as_raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
1482 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1483 check_added_monitors!(nodes[1], 1);
1485 expect_pending_htlcs_forwardable!(nodes[1]);
1486 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1488 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_1, 1_000_000);
1489 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1493 fn test_monitor_update_fail_claim() {
1494 // Basic test for monitor update failures when processing claim_funds calls.
1495 // We set up a simple 3-node network, sending a payment from A to B and failing B's monitor
1496 // update to claim the payment. We then send a payment C->B->A, making the forward of this
1497 // payment from B to A fail due to the paused channel. Finally, we restore the channel monitor
1498 // updating and claim the payment on B.
1499 let chanmon_cfgs = create_chanmon_cfgs(3);
1500 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1501 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1502 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1503 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1504 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1505 let logger = test_utils::TestLogger::new();
1507 // Rebalance a bit so that we can send backwards from 3 to 2.
1508 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1510 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1512 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1513 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1514 check_added_monitors!(nodes[1], 1);
1516 let (_, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1518 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1519 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1520 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1521 check_added_monitors!(nodes[2], 1);
1524 // Successfully update the monitor on the 1<->2 channel, but the 0<->1 channel should still be
1525 // paused, so forward shouldn't succeed until we call channel_monitor_updated().
1526 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1528 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1529 assert_eq!(events.len(), 1);
1530 let payment_event = SendEvent::from_event(events.pop().unwrap());
1531 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1532 let events = nodes[1].node.get_and_clear_pending_msg_events();
1533 assert_eq!(events.len(), 0);
1534 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1535 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false, true);
1537 let bs_fail_update = get_htlc_update_msgs!(nodes[1], nodes[2].node.get_our_node_id());
1538 nodes[2].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_fail_update.update_fail_htlcs[0]);
1539 commitment_signed_dance!(nodes[2], nodes[1], bs_fail_update.commitment_signed, false, true);
1541 let msg_events = nodes[2].node.get_and_clear_pending_msg_events();
1542 assert_eq!(msg_events.len(), 1);
1543 match msg_events[0] {
1544 MessageSendEvent::PaymentFailureNetworkUpdate { update: msgs::HTLCFailChannelUpdate::ChannelUpdateMessage { ref msg }} => {
1545 assert_eq!(msg.contents.short_channel_id, chan_1.0.contents.short_channel_id);
1546 assert_eq!(msg.contents.flags & 2, 2); // temp disabled
1548 _ => panic!("Unexpected event"),
1551 let events = nodes[2].node.get_and_clear_pending_events();
1552 assert_eq!(events.len(), 1);
1553 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1554 assert_eq!(payment_hash, payment_hash_2);
1555 assert!(!rejected_by_dest);
1556 } else { panic!("Unexpected event!"); }
1558 // Now restore monitor updating on the 0<->1 channel and claim the funds on B.
1559 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1560 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1561 check_added_monitors!(nodes[1], 0);
1563 let bs_fulfill_update = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1564 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_fulfill_update.update_fulfill_htlcs[0]);
1565 commitment_signed_dance!(nodes[0], nodes[1], bs_fulfill_update.commitment_signed, false);
1567 let events = nodes[0].node.get_and_clear_pending_events();
1568 assert_eq!(events.len(), 1);
1569 if let Event::PaymentSent { payment_preimage, .. } = events[0] {
1570 assert_eq!(payment_preimage, payment_preimage_1);
1571 } else { panic!("Unexpected event!"); }
1575 fn test_monitor_update_on_pending_forwards() {
1576 // Basic test for monitor update failures when processing pending HTLC fail/add forwards.
1577 // We do this with a simple 3-node network, sending a payment from A to C and one from C to A.
1578 // The payment from A to C will be failed by C and pending a back-fail to A, while the payment
1579 // from C to A will be pending a forward to A.
1580 let chanmon_cfgs = create_chanmon_cfgs(3);
1581 let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1582 let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1583 let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1584 let chan_1 = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known());
1585 create_announced_chan_between_nodes(&nodes, 1, 2, InitFeatures::known(), InitFeatures::known());
1586 let logger = test_utils::TestLogger::new();
1588 // Rebalance a bit so that we can send backwards from 3 to 1.
1589 send_payment(&nodes[0], &[&nodes[1], &nodes[2]], 5000000, 5_000_000);
1591 let (_, payment_hash_1) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1000000);
1592 assert!(nodes[2].node.fail_htlc_backwards(&payment_hash_1, &None));
1593 expect_pending_htlcs_forwardable!(nodes[2]);
1594 check_added_monitors!(nodes[2], 1);
1596 let cs_fail_update = get_htlc_update_msgs!(nodes[2], nodes[1].node.get_our_node_id());
1597 nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &cs_fail_update.update_fail_htlcs[0]);
1598 commitment_signed_dance!(nodes[1], nodes[2], cs_fail_update.commitment_signed, true, true);
1599 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1601 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1603 let net_graph_msg_handler = &nodes[2].net_graph_msg_handler;
1604 let route = get_route(&nodes[2].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[0].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1605 nodes[2].node.send_payment(&route, payment_hash_2, &None).unwrap();
1606 check_added_monitors!(nodes[2], 1);
1609 let mut events = nodes[2].node.get_and_clear_pending_msg_events();
1610 assert_eq!(events.len(), 1);
1611 let payment_event = SendEvent::from_event(events.pop().unwrap());
1612 nodes[1].node.handle_update_add_htlc(&nodes[2].node.get_our_node_id(), &payment_event.msgs[0]);
1613 commitment_signed_dance!(nodes[1], nodes[2], payment_event.commitment_msg, false);
1615 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1616 expect_pending_htlcs_forwardable!(nodes[1]);
1617 check_added_monitors!(nodes[1], 1);
1618 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1619 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1621 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1622 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_1.2).unwrap().clone();
1623 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1624 check_added_monitors!(nodes[1], 0);
1626 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1627 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fail_htlcs[0]);
1628 nodes[0].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_add_htlcs[0]);
1629 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false, true);
1631 let events = nodes[0].node.get_and_clear_pending_events();
1632 assert_eq!(events.len(), 2);
1633 if let Event::PaymentFailed { payment_hash, rejected_by_dest, .. } = events[0] {
1634 assert_eq!(payment_hash, payment_hash_1);
1635 assert!(rejected_by_dest);
1636 } else { panic!("Unexpected event!"); }
1638 Event::PendingHTLCsForwardable { .. } => { },
1639 _ => panic!("Unexpected event"),
1641 nodes[0].node.process_pending_htlc_forwards();
1642 expect_payment_received!(nodes[0], payment_hash_2, 1000000);
1644 claim_payment(&nodes[2], &[&nodes[1], &nodes[0]], payment_preimage_2, 1_000_000);
1648 fn monitor_update_claim_fail_no_response() {
1649 // Test for claim_funds resulting in both a monitor update failure and no message response (due
1650 // to channel being AwaitingRAA).
1651 // Backported from chanmon_fail_consistency fuzz tests as an unmerged version of the handling
1653 let chanmon_cfgs = create_chanmon_cfgs(2);
1654 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1655 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1656 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1657 let channel_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).2;
1658 let logger = test_utils::TestLogger::new();
1660 // Forward a payment for B to claim
1661 let (payment_preimage_1, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
1663 // Now start forwarding a second payment, skipping the last RAA so B is in AwaitingRAA
1664 let (payment_preimage_2, payment_hash_2) = get_payment_preimage_hash!(nodes[0]);
1666 let net_graph_msg_handler = &nodes[0].net_graph_msg_handler;
1667 let route = get_route(&nodes[0].node.get_our_node_id(), &net_graph_msg_handler.network_graph.read().unwrap(), &nodes[1].node.get_our_node_id(), None, &Vec::new(), 1000000, TEST_FINAL_CLTV, &logger).unwrap();
1668 nodes[0].node.send_payment(&route, payment_hash_2, &None).unwrap();
1669 check_added_monitors!(nodes[0], 1);
1672 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1673 assert_eq!(events.len(), 1);
1674 let payment_event = SendEvent::from_event(events.pop().unwrap());
1675 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
1676 let as_raa = commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false, true, false, true);
1678 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1679 assert!(nodes[1].node.claim_funds(payment_preimage_1, &None, 1_000_000));
1680 check_added_monitors!(nodes[1], 1);
1681 let events = nodes[1].node.get_and_clear_pending_msg_events();
1682 assert_eq!(events.len(), 0);
1683 nodes[1].logger.assert_log("lightning::ln::channelmanager".to_string(), "Temporary failure claiming HTLC, treating as success: Failed to update ChannelMonitor".to_string(), 1);
1685 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1686 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1687 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1688 check_added_monitors!(nodes[1], 0);
1689 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1691 nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &as_raa);
1692 check_added_monitors!(nodes[1], 1);
1693 expect_pending_htlcs_forwardable!(nodes[1]);
1694 expect_payment_received!(nodes[1], payment_hash_2, 1000000);
1696 let bs_updates = get_htlc_update_msgs!(nodes[1], nodes[0].node.get_our_node_id());
1697 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &bs_updates.update_fulfill_htlcs[0]);
1698 commitment_signed_dance!(nodes[0], nodes[1], bs_updates.commitment_signed, false);
1700 let events = nodes[0].node.get_and_clear_pending_events();
1701 assert_eq!(events.len(), 1);
1703 Event::PaymentSent { ref payment_preimage } => {
1704 assert_eq!(*payment_preimage, payment_preimage_1);
1706 _ => panic!("Unexpected event"),
1709 claim_payment(&nodes[0], &[&nodes[1]], payment_preimage_2, 1_000_000);
1712 // confirm_a_first and restore_b_before_conf are wholly unrelated to earlier bools and
1713 // restore_b_before_conf has no meaning if !confirm_a_first
1714 fn do_during_funding_monitor_fail(confirm_a_first: bool, restore_b_before_conf: bool) {
1715 // Test that if the monitor update generated by funding_transaction_generated fails we continue
1716 // the channel setup happily after the update is restored.
1717 let chanmon_cfgs = create_chanmon_cfgs(2);
1718 let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
1719 let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
1720 let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
1722 nodes[0].node.create_channel(nodes[1].node.get_our_node_id(), 100000, 10001, 43, None).unwrap();
1723 nodes[1].node.handle_open_channel(&nodes[0].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[0], MessageSendEvent::SendOpenChannel, nodes[1].node.get_our_node_id()));
1724 nodes[0].node.handle_accept_channel(&nodes[1].node.get_our_node_id(), InitFeatures::known(), &get_event_msg!(nodes[1], MessageSendEvent::SendAcceptChannel, nodes[0].node.get_our_node_id()));
1726 let (temporary_channel_id, funding_tx, funding_output) = create_funding_transaction(&nodes[0], 100000, 43);
1728 nodes[0].node.funding_transaction_generated(&temporary_channel_id, funding_output);
1729 check_added_monitors!(nodes[0], 0);
1731 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1732 let funding_created_msg = get_event_msg!(nodes[0], MessageSendEvent::SendFundingCreated, nodes[1].node.get_our_node_id());
1733 let channel_id = OutPoint { txid: funding_created_msg.funding_txid, index: funding_created_msg.funding_output_index }.to_channel_id();
1734 nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msg);
1735 check_added_monitors!(nodes[1], 1);
1737 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Err(ChannelMonitorUpdateErr::TemporaryFailure);
1738 nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id()));
1739 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1740 nodes[0].logger.assert_log("lightning::ln::channelmanager".to_string(), "Failed to update ChannelMonitor".to_string(), 1);
1741 check_added_monitors!(nodes[0], 1);
1742 assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1743 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
1744 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1745 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1746 check_added_monitors!(nodes[0], 0);
1748 let events = nodes[0].node.get_and_clear_pending_events();
1749 assert_eq!(events.len(), 1);
1751 Event::FundingBroadcastSafe { ref funding_txo, user_channel_id } => {
1752 assert_eq!(user_channel_id, 43);
1753 assert_eq!(*funding_txo, funding_output);
1755 _ => panic!("Unexpected event"),
1758 if confirm_a_first {
1759 confirm_transaction(&nodes[0], &funding_tx);
1760 nodes[1].node.handle_funding_locked(&nodes[0].node.get_our_node_id(), &get_event_msg!(nodes[0], MessageSendEvent::SendFundingLocked, nodes[1].node.get_our_node_id()));
1762 assert!(!restore_b_before_conf);
1763 confirm_transaction(&nodes[1], &funding_tx);
1764 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1767 // Make sure nodes[1] isn't stupid enough to re-send the FundingLocked on reconnect
1768 nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
1769 nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
1770 reconnect_nodes(&nodes[0], &nodes[1], (false, confirm_a_first), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1771 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
1772 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1774 if !restore_b_before_conf {
1775 confirm_transaction(&nodes[1], &funding_tx);
1776 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1777 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1780 *nodes[1].chain_monitor.update_ret.lock().unwrap() = Ok(());
1781 let (outpoint, latest_update) = nodes[1].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&channel_id).unwrap().clone();
1782 nodes[1].node.channel_monitor_updated(&outpoint, latest_update);
1783 check_added_monitors!(nodes[1], 0);
1785 let (channel_id, (announcement, as_update, bs_update)) = if !confirm_a_first {
1786 nodes[0].node.handle_funding_locked(&nodes[1].node.get_our_node_id(), &get_event_msg!(nodes[1], MessageSendEvent::SendFundingLocked, nodes[0].node.get_our_node_id()));
1788 confirm_transaction(&nodes[0], &funding_tx);
1789 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[1], &nodes[0]);
1790 (channel_id, create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &funding_locked))
1792 if restore_b_before_conf {
1793 confirm_transaction(&nodes[1], &funding_tx);
1795 let (funding_locked, channel_id) = create_chan_between_nodes_with_value_confirm_second(&nodes[0], &nodes[1]);
1796 (channel_id, create_chan_between_nodes_with_value_b(&nodes[1], &nodes[0], &funding_locked))
1798 for node in nodes.iter() {
1799 assert!(node.net_graph_msg_handler.handle_channel_announcement(&announcement).unwrap());
1800 node.net_graph_msg_handler.handle_channel_update(&as_update).unwrap();
1801 node.net_graph_msg_handler.handle_channel_update(&bs_update).unwrap();
1804 send_payment(&nodes[0], &[&nodes[1]], 8000000, 8_000_000);
1805 close_channel(&nodes[0], &nodes[1], &channel_id, funding_tx, true);
1809 fn during_funding_monitor_fail() {
1810 do_during_funding_monitor_fail(true, true);
1811 do_during_funding_monitor_fail(true, false);
1812 do_during_funding_monitor_fail(false, false);
1816 fn test_path_paused_mpp() {
1817 // Simple test of sending a multi-part payment where one path is currently blocked awaiting
1819 let chanmon_cfgs = create_chanmon_cfgs(4);
1820 let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
1821 let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
1822 let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
1824 let chan_1_id = create_announced_chan_between_nodes(&nodes, 0, 1, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1825 let (chan_2_ann, _, chan_2_id, _) = create_announced_chan_between_nodes(&nodes, 0, 2, InitFeatures::known(), InitFeatures::known());
1826 let chan_3_id = create_announced_chan_between_nodes(&nodes, 1, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1827 let chan_4_id = create_announced_chan_between_nodes(&nodes, 2, 3, InitFeatures::known(), InitFeatures::known()).0.contents.short_channel_id;
1828 let logger = test_utils::TestLogger::new();
1830 let (payment_preimage, payment_hash) = get_payment_preimage_hash!(&nodes[0]);
1831 let payment_secret = PaymentSecret([0xdb; 32]);
1832 let mut route = get_route(&nodes[0].node.get_our_node_id(), &nodes[0].net_graph_msg_handler.network_graph.read().unwrap(), &nodes[3].node.get_our_node_id(), None, &[], 100000, TEST_FINAL_CLTV, &logger).unwrap();
1834 // Set us up to take multiple routes, one 0 -> 1 -> 3 and one 0 -> 2 -> 3:
1835 let path = route.paths[0].clone();
1836 route.paths.push(path);
1837 route.paths[0][0].pubkey = nodes[1].node.get_our_node_id();
1838 route.paths[0][0].short_channel_id = chan_1_id;
1839 route.paths[0][1].short_channel_id = chan_3_id;
1840 route.paths[1][0].pubkey = nodes[2].node.get_our_node_id();
1841 route.paths[1][0].short_channel_id = chan_2_ann.contents.short_channel_id;
1842 route.paths[1][1].short_channel_id = chan_4_id;
1844 // Set it so that the first monitor update (for the path 0 -> 1 -> 3) succeeds, but the second
1845 // (for the path 0 -> 2 -> 3) fails.
1846 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
1847 *nodes[0].chain_monitor.next_update_ret.lock().unwrap() = Some(Err(ChannelMonitorUpdateErr::TemporaryFailure));
1849 // Now check that we get the right return value, indicating that the first path succeeded but
1850 // the second got a MonitorUpdateFailed err. This implies PaymentSendFailure::PartialFailure as
1851 // some paths succeeded, preventing retry.
1852 if let Err(PaymentSendFailure::PartialFailure(results)) = nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret)) {
1853 assert_eq!(results.len(), 2);
1854 if let Ok(()) = results[0] {} else { panic!(); }
1855 if let Err(APIError::MonitorUpdateFailed) = results[1] {} else { panic!(); }
1856 } else { panic!(); }
1857 check_added_monitors!(nodes[0], 2);
1858 *nodes[0].chain_monitor.update_ret.lock().unwrap() = Ok(());
1860 // Pass the first HTLC of the payment along to nodes[3].
1861 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1862 assert_eq!(events.len(), 1);
1863 pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 0, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), false);
1865 // And check that, after we successfully update the monitor for chan_2 we can pass the second
1866 // HTLC along to nodes[3] and claim the whole payment back to nodes[0].
1867 let (outpoint, latest_update) = nodes[0].chain_monitor.latest_monitor_update_id.lock().unwrap().get(&chan_2_id).unwrap().clone();
1868 nodes[0].node.channel_monitor_updated(&outpoint, latest_update);
1869 let mut events = nodes[0].node.get_and_clear_pending_msg_events();
1870 assert_eq!(events.len(), 1);
1871 pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 200_000, payment_hash.clone(), Some(payment_secret), events.pop().unwrap(), true);
1873 claim_payment_along_route_with_secret(&nodes[0], &[&[&nodes[1], &nodes[3]], &[&nodes[2], &nodes[3]]], false, payment_preimage, Some(payment_secret), 200_000);