]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
Merge pull request #2575 from tnull/2023-09-fix-debug-panic
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::{CLOSED_CHANNEL_UPDATE_ID, ChannelMonitor};
15 use crate::sign::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
19 use crate::ln::msgs;
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::util::test_channel_signer::TestChannelSigner;
22 use crate::util::test_utils;
23 use crate::util::errors::APIError;
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26 use crate::util::string::UntrustedString;
27
28 use bitcoin::{PackedLockTime, Transaction, TxOut};
29 use bitcoin::hash_types::BlockHash;
30
31 use crate::prelude::*;
32 use core::default::Default;
33 use crate::sync::Mutex;
34
35 use crate::ln::functional_test_utils::*;
36
37 #[test]
38 fn test_funding_peer_disconnect() {
39         // Test that we can lock in our funding tx while disconnected
40         let chanmon_cfgs = create_chanmon_cfgs(2);
41         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
42         let persister;
43         let new_chain_monitor;
44
45         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
46         let nodes_0_deserialized;
47         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
48         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
49
50         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
51         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
52
53         confirm_transaction(&nodes[0], &tx);
54         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
55         assert!(events_1.is_empty());
56
57         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
58         reconnect_args.send_channel_ready.1 = true;
59         reconnect_nodes(reconnect_args);
60
61         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
62         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
63
64         confirm_transaction(&nodes[1], &tx);
65         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
66         assert!(events_2.is_empty());
67
68         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
69                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
70         }, true).unwrap();
71         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
72         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
73                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
74         }, false).unwrap();
75         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
76
77         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
78         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
79         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
80         assert_eq!(events_3.len(), 1);
81         let as_channel_ready = match events_3[0] {
82                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
83                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
84                         msg.clone()
85                 },
86                 _ => panic!("Unexpected event {:?}", events_3[0]),
87         };
88
89         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
90         // announcement_signatures as well as channel_update.
91         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
92         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
93         assert_eq!(events_4.len(), 3);
94         let chan_id;
95         let bs_channel_ready = match events_4[0] {
96                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
97                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
98                         chan_id = msg.channel_id;
99                         msg.clone()
100                 },
101                 _ => panic!("Unexpected event {:?}", events_4[0]),
102         };
103         let bs_announcement_sigs = match events_4[1] {
104                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
105                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
106                         msg.clone()
107                 },
108                 _ => panic!("Unexpected event {:?}", events_4[1]),
109         };
110         match events_4[2] {
111                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
112                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
113                 },
114                 _ => panic!("Unexpected event {:?}", events_4[2]),
115         }
116
117         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
118         // generates a duplicative private channel_update
119         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
120         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
121         assert_eq!(events_5.len(), 1);
122         match events_5[0] {
123                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
124                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
125                 },
126                 _ => panic!("Unexpected event {:?}", events_5[0]),
127         };
128
129         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
130         // announcement_signatures.
131         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
132         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
133         assert_eq!(events_6.len(), 1);
134         let as_announcement_sigs = match events_6[0] {
135                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
136                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
137                         msg.clone()
138                 },
139                 _ => panic!("Unexpected event {:?}", events_6[0]),
140         };
141         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
142         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
143
144         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
145         // broadcast the channel announcement globally, as well as re-send its (now-public)
146         // channel_update.
147         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
148         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
149         assert_eq!(events_7.len(), 1);
150         let (chan_announcement, as_update) = match events_7[0] {
151                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
152                         (msg.clone(), update_msg.clone().unwrap())
153                 },
154                 _ => panic!("Unexpected event {:?}", events_7[0]),
155         };
156
157         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
158         // same channel_announcement.
159         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
160         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
161         assert_eq!(events_8.len(), 1);
162         let bs_update = match events_8[0] {
163                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
164                         assert_eq!(*msg, chan_announcement);
165                         update_msg.clone().unwrap()
166                 },
167                 _ => panic!("Unexpected event {:?}", events_8[0]),
168         };
169
170         // Provide the channel announcement and public updates to the network graph
171         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
172         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
173         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
174
175         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
176         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
177         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
178
179         // Check that after deserialization and reconnection we can still generate an identical
180         // channel_announcement from the cached signatures.
181         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
182
183         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
184
185         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
186
187         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
188 }
189
190 #[test]
191 fn test_no_txn_manager_serialize_deserialize() {
192         let chanmon_cfgs = create_chanmon_cfgs(2);
193         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
194         let persister;
195         let new_chain_monitor;
196
197         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
198         let nodes_0_deserialized;
199         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
200
201         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
202
203         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
204
205         let chan_0_monitor_serialized =
206                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
207         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
208
209         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
210                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
211         }, true).unwrap();
212         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
213         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
214                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
215         }, false).unwrap();
216         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
217
218         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
219         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
220         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
221         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
222
223         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
224         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
225         for node in nodes.iter() {
226                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
227                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
228                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
229         }
230
231         send_payment(&nodes[0], &[&nodes[1]], 1000000);
232 }
233
234 #[test]
235 fn test_manager_serialize_deserialize_events() {
236         // This test makes sure the events field in ChannelManager survives de/serialization
237         let chanmon_cfgs = create_chanmon_cfgs(2);
238         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
239         let persister;
240         let new_chain_monitor;
241
242         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
243         let nodes_0_deserialized;
244         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
245
246         // Start creating a channel, but stop right before broadcasting the funding transaction
247         let channel_value = 100000;
248         let push_msat = 10001;
249         let node_a = nodes.remove(0);
250         let node_b = nodes.remove(0);
251         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
252         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
253         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
254
255         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
256
257         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
258         check_added_monitors!(node_a, 0);
259
260         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
261         {
262                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
263                 assert_eq!(added_monitors.len(), 1);
264                 assert_eq!(added_monitors[0].0, funding_output);
265                 added_monitors.clear();
266         }
267
268         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
269         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
270         {
271                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
272                 assert_eq!(added_monitors.len(), 1);
273                 assert_eq!(added_monitors[0].0, funding_output);
274                 added_monitors.clear();
275         }
276         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
277
278         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
279         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
280
281         nodes.push(node_a);
282         nodes.push(node_b);
283
284         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
285         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
286         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
287
288         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
289
290         // After deserializing, make sure the funding_transaction is still held by the channel manager
291         let events_4 = nodes[0].node.get_and_clear_pending_events();
292         assert_eq!(events_4.len(), 0);
293         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
294         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
295
296         // Make sure the channel is functioning as though the de/serialization never happened
297         assert_eq!(nodes[0].node.list_channels().len(), 1);
298
299         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
300                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
301         }, true).unwrap();
302         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
303         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
304                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
305         }, false).unwrap();
306         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
307
308         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
309         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
310         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
311         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
312
313         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
314         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
315         for node in nodes.iter() {
316                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
317                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
318                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
319         }
320
321         send_payment(&nodes[0], &[&nodes[1]], 1000000);
322 }
323
324 #[test]
325 fn test_simple_manager_serialize_deserialize() {
326         let chanmon_cfgs = create_chanmon_cfgs(2);
327         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
328         let persister;
329         let new_chain_monitor;
330
331         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
332         let nodes_0_deserialized;
333         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
334         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
335
336         let (our_payment_preimage, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
337         let (_, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
338
339         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
340
341         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
342         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
343
344         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
345
346         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
347         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
348 }
349
350 #[test]
351 fn test_manager_serialize_deserialize_inconsistent_monitor() {
352         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
353         let chanmon_cfgs = create_chanmon_cfgs(4);
354         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
355         let logger;
356         let fee_estimator;
357         let persister;
358         let new_chain_monitor;
359
360         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
361         let nodes_0_deserialized;
362         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
363
364         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
365         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
366         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
367
368         let mut node_0_stale_monitors_serialized = Vec::new();
369         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
370                 let mut writer = test_utils::TestVecWriter(Vec::new());
371                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
372                 node_0_stale_monitors_serialized.push(writer.0);
373         }
374
375         let (our_payment_preimage, ..) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
376
377         // Serialize the ChannelManager here, but the monitor we keep up-to-date
378         let nodes_0_serialized = nodes[0].node.encode();
379
380         route_payment(&nodes[0], &[&nodes[3]], 1000000);
381         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
382         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
383         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
384
385         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
386         // nodes[3])
387         let mut node_0_monitors_serialized = Vec::new();
388         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
389                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
390         }
391
392         logger = test_utils::TestLogger::new();
393         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
394         persister = test_utils::TestPersister::new();
395         let keys_manager = &chanmon_cfgs[0].keys_manager;
396         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
397         nodes[0].chain_monitor = &new_chain_monitor;
398
399
400         let mut node_0_stale_monitors = Vec::new();
401         for serialized in node_0_stale_monitors_serialized.iter() {
402                 let mut read = &serialized[..];
403                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
404                 assert!(read.is_empty());
405                 node_0_stale_monitors.push(monitor);
406         }
407
408         let mut node_0_monitors = Vec::new();
409         for serialized in node_0_monitors_serialized.iter() {
410                 let mut read = &serialized[..];
411                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
412                 assert!(read.is_empty());
413                 node_0_monitors.push(monitor);
414         }
415
416         let mut nodes_0_read = &nodes_0_serialized[..];
417         if let Err(msgs::DecodeError::InvalidValue) =
418                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
419                 default_config: UserConfig::default(),
420                 entropy_source: keys_manager,
421                 node_signer: keys_manager,
422                 signer_provider: keys_manager,
423                 fee_estimator: &fee_estimator,
424                 router: &nodes[0].router,
425                 chain_monitor: nodes[0].chain_monitor,
426                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
427                 logger: &logger,
428                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
429         }) { } else {
430                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
431         };
432
433         let mut nodes_0_read = &nodes_0_serialized[..];
434         let (_, nodes_0_deserialized_tmp) =
435                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
436                 default_config: UserConfig::default(),
437                 entropy_source: keys_manager,
438                 node_signer: keys_manager,
439                 signer_provider: keys_manager,
440                 fee_estimator: &fee_estimator,
441                 router: nodes[0].router,
442                 chain_monitor: nodes[0].chain_monitor,
443                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
444                 logger: &logger,
445                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
446         }).unwrap();
447         nodes_0_deserialized = nodes_0_deserialized_tmp;
448         assert!(nodes_0_read.is_empty());
449
450         for monitor in node_0_monitors.drain(..) {
451                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
452                         Ok(ChannelMonitorUpdateStatus::Completed));
453                 check_added_monitors!(nodes[0], 1);
454         }
455         nodes[0].node = &nodes_0_deserialized;
456
457         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[3].node.get_our_node_id()], 100000);
458         { // Channel close should result in a commitment tx
459                 nodes[0].node.timer_tick_occurred();
460                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
461                 assert_eq!(txn.len(), 1);
462                 check_spends!(txn[0], funding_tx);
463                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
464         }
465         check_added_monitors!(nodes[0], 1);
466
467         // nodes[1] and nodes[2] have no lost state with nodes[0]...
468         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
469         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
470         //... and we can even still claim the payment!
471         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
472
473         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
474                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
475         }, true).unwrap();
476         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
477         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
478                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
479         }, false).unwrap();
480         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
481         let mut found_err = false;
482         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
483                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
484                         match action {
485                                 &ErrorAction::SendErrorMessage { ref msg } => {
486                                         assert_eq!(msg.channel_id, channel_id);
487                                         assert!(!found_err);
488                                         found_err = true;
489                                 },
490                                 _ => panic!("Unexpected event!"),
491                         }
492                 }
493         }
494         assert!(found_err);
495 }
496
497 fn do_test_data_loss_protect(reconnect_panicing: bool) {
498         // When we get a data_loss_protect proving we're behind, we immediately panic as the
499         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
500         // panic message informs the user they should force-close without broadcasting, which is tested
501         // if `reconnect_panicing` is not set.
502         let mut chanmon_cfgs = create_chanmon_cfgs(2);
503         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
504         // during signing due to revoked tx
505         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
506         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
507         let persister;
508         let new_chain_monitor;
509
510         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
511         let nodes_0_deserialized;
512
513         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
514
515         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
516
517         // Cache node A state before any channel update
518         let previous_node_state = nodes[0].node.encode();
519         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
520
521         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
522         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
523
524         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
525         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
526
527         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
528
529         if reconnect_panicing {
530                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
531                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
532                 }, true).unwrap();
533                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
534                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
535                 }, false).unwrap();
536
537                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
538
539                 // Check we close channel detecting A is fallen-behind
540                 // Check that we sent the warning message when we detected that A has fallen behind,
541                 // and give the possibility for A to recover from the warning.
542                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
543                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
544                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
545
546                 {
547                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
548                         // The node B should not broadcast the transaction to force close the channel!
549                         assert!(node_txn.is_empty());
550                 }
551
552                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
553                 // Check A panics upon seeing proof it has fallen behind.
554                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
555                 return; // By this point we should have panic'ed!
556         }
557
558         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
559         check_added_monitors!(nodes[0], 1);
560         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 1000000);
561         {
562                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
563                 assert_eq!(node_txn.len(), 0);
564         }
565
566         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
567                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
568                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
569                         match action {
570                                 &ErrorAction::SendErrorMessage { ref msg } => {
571                                         assert_eq!(msg.data, "Channel force-closed");
572                                 },
573                                 _ => panic!("Unexpected event!"),
574                         }
575                 } else {
576                         panic!("Unexpected event {:?}", msg)
577                 }
578         }
579
580         // after the warning message sent by B, we should not able to
581         // use the channel, or reconnect with success to the channel.
582         assert!(nodes[0].node.list_usable_channels().is_empty());
583         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
584                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
585         }, true).unwrap();
586         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
587                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
588         }, false).unwrap();
589         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
590
591         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
592         let mut err_msgs_0 = Vec::with_capacity(1);
593         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
594                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
595                         match action {
596                                 &ErrorAction::SendErrorMessage { ref msg } => {
597                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
598                                         err_msgs_0.push(msg.clone());
599                                 },
600                                 _ => panic!("Unexpected event!"),
601                         }
602                 } else {
603                         panic!("Unexpected event!");
604                 }
605         }
606         assert_eq!(err_msgs_0.len(), 1);
607         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
608         assert!(nodes[1].node.list_usable_channels().is_empty());
609         check_added_monitors!(nodes[1], 1);
610         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
611                 , [nodes[0].node.get_our_node_id()], 1000000);
612         check_closed_broadcast!(nodes[1], false);
613 }
614
615 #[test]
616 #[should_panic]
617 fn test_data_loss_protect_showing_stale_state_panics() {
618         do_test_data_loss_protect(true);
619 }
620
621 #[test]
622 fn test_force_close_without_broadcast() {
623         do_test_data_loss_protect(false);
624 }
625
626 #[test]
627 fn test_forwardable_regen() {
628         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
629         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
630         // HTLCs.
631         // We test it for both payment receipt and payment forwarding.
632
633         let chanmon_cfgs = create_chanmon_cfgs(3);
634         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
635         let persister;
636         let new_chain_monitor;
637         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
638         let nodes_1_deserialized;
639         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
640         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
641         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
642
643         // First send a payment to nodes[1]
644         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
645         nodes[0].node.send_payment_with_route(&route, payment_hash,
646                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
647         check_added_monitors!(nodes[0], 1);
648
649         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
650         assert_eq!(events.len(), 1);
651         let payment_event = SendEvent::from_event(events.pop().unwrap());
652         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
653         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
654
655         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
656
657         // Next send a payment which is forwarded by nodes[1]
658         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
659         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
660                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
661         check_added_monitors!(nodes[0], 1);
662
663         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
664         assert_eq!(events.len(), 1);
665         let payment_event = SendEvent::from_event(events.pop().unwrap());
666         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
667         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
668
669         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
670         // generated
671         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
672
673         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
674         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
675         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
676
677         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
678         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
679         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
680
681         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
682         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
683         // the commitment state.
684         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
685         reconnect_args.send_channel_ready = (true, true);
686         reconnect_nodes(reconnect_args);
687
688         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
689
690         expect_pending_htlcs_forwardable!(nodes[1]);
691         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
692         check_added_monitors!(nodes[1], 1);
693
694         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
695         assert_eq!(events.len(), 1);
696         let payment_event = SendEvent::from_event(events.pop().unwrap());
697         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
698         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
699         expect_pending_htlcs_forwardable!(nodes[2]);
700         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
701
702         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
703         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
704 }
705
706 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
707         // Test what happens if a node receives an MPP payment, claims it, but crashes before
708         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
709         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
710         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
711         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
712         // not have the preimage tied to the still-pending HTLC.
713         //
714         // To get to the correct state, on startup we should propagate the preimage to the
715         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
716         // receiving the preimage without a state update.
717         //
718         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
719         // definitely claimed.
720         let chanmon_cfgs = create_chanmon_cfgs(4);
721         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
722         let persister;
723         let new_chain_monitor;
724
725         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
726         let nodes_3_deserialized;
727
728         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
729
730         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
731         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
732         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
733         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
734
735         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
736         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
737         assert_eq!(route.paths.len(), 2);
738         route.paths.sort_by(|path_a, _| {
739                 // Sort the path so that the path through nodes[1] comes first
740                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
741                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
742         });
743
744         nodes[0].node.send_payment_with_route(&route, payment_hash,
745                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
746         check_added_monitors!(nodes[0], 2);
747
748         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
749         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
750         assert_eq!(send_events.len(), 2);
751         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
752         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
753         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
754         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
755
756         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
757         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
758         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
759         if !persist_both_monitors {
760                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
761                         if outpoint.to_channel_id() == chan_id_not_persisted {
762                                 assert!(original_monitor.0.is_empty());
763                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
764                         }
765                 }
766         }
767
768         let original_manager = nodes[3].node.encode();
769
770         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
771
772         nodes[3].node.claim_funds(payment_preimage);
773         check_added_monitors!(nodes[3], 2);
774         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
775
776         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
777         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
778         // with the old ChannelManager.
779         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
780         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
781                 if outpoint.to_channel_id() == chan_id_persisted {
782                         assert!(updated_monitor.0.is_empty());
783                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
784                 }
785         }
786         // If `persist_both_monitors` is set, get the second monitor here as well
787         if persist_both_monitors {
788                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
789                         if outpoint.to_channel_id() == chan_id_not_persisted {
790                                 assert!(original_monitor.0.is_empty());
791                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
792                         }
793                 }
794         }
795
796         // Now restart nodes[3].
797         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
798
799         // On startup the preimage should have been copied into the non-persisted monitor:
800         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
801         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
802
803         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
804         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
805
806         // During deserialization, we should have closed one channel and broadcast its latest
807         // commitment transaction. We should also still have the original PaymentClaimable event we
808         // never finished processing.
809         let events = nodes[3].node.get_and_clear_pending_events();
810         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
811         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
812         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
813         if persist_both_monitors {
814                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
815                 check_added_monitors(&nodes[3], 2);
816         } else {
817                 check_added_monitors(&nodes[3], 1);
818         }
819
820         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
821         // ChannelManager prior to handling the original one.
822         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
823                 events[if persist_both_monitors { 3 } else { 2 }]
824         {
825                 assert_eq!(payment_hash, our_payment_hash);
826         } else { panic!(); }
827
828         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
829         if !persist_both_monitors {
830                 // If one of the two channels is still live, reveal the payment preimage over it.
831
832                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
833                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
834                 }, true).unwrap();
835                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
836                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
837                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
838                 }, false).unwrap();
839                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
840
841                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
842                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
843                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
844
845                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
846
847                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
848                 // claim should fly.
849                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
850                 check_added_monitors!(nodes[3], 1);
851                 assert_eq!(ds_msgs.len(), 2);
852                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
853
854                 let cs_updates = match ds_msgs[1] {
855                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
856                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
857                                 check_added_monitors!(nodes[2], 1);
858                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
859                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
860                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
861                                 cs_updates
862                         }
863                         _ => panic!(),
864                 };
865
866                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
867                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
868                 expect_payment_sent!(nodes[0], payment_preimage);
869         }
870 }
871
872 #[test]
873 fn test_partial_claim_before_restart() {
874         do_test_partial_claim_before_restart(false);
875         do_test_partial_claim_before_restart(true);
876 }
877
878 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
879         if !use_cs_commitment { assert!(!claim_htlc); }
880         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
881         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
882         // it back until the ChannelMonitor decides the fate of the HTLC.
883         // This was never an issue, but it may be easy to regress here going forward.
884         let chanmon_cfgs = create_chanmon_cfgs(3);
885         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
886         let persister;
887         let new_chain_monitor;
888
889         let mut intercept_forwards_config = test_default_channel_config();
890         intercept_forwards_config.accept_intercept_htlcs = true;
891         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
892         let nodes_1_deserialized;
893
894         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
895
896         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
897         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
898
899         let intercept_scid = nodes[1].node.get_intercept_scid();
900
901         let (mut route, payment_hash, payment_preimage, payment_secret) =
902                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
903         if use_intercept {
904                 route.paths[0].hops[1].short_channel_id = intercept_scid;
905         }
906         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
907         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
908         nodes[0].node.send_payment_with_route(&route, payment_hash,
909                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
910         check_added_monitors!(nodes[0], 1);
911
912         let payment_event = SendEvent::from_node(&nodes[0]);
913         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
914         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
915
916         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
917         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
918         // present when we serialized.
919         let node_encoded = nodes[1].node.encode();
920
921         let mut intercept_id = None;
922         let mut expected_outbound_amount_msat = None;
923         if use_intercept {
924                 let events = nodes[1].node.get_and_clear_pending_events();
925                 assert_eq!(events.len(), 1);
926                 match events[0] {
927                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
928                                 intercept_id = Some(ev_id);
929                                 expected_outbound_amount_msat = Some(ev_amt);
930                         },
931                         _ => panic!()
932                 }
933                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
934                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
935         }
936
937         expect_pending_htlcs_forwardable!(nodes[1]);
938
939         let payment_event = SendEvent::from_node(&nodes[1]);
940         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
941         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
942         check_added_monitors!(nodes[2], 1);
943
944         if claim_htlc {
945                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
946                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
947         }
948         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
949
950         let _ = nodes[2].node.get_and_clear_pending_msg_events();
951
952         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
953         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
954         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
955
956         check_added_monitors!(nodes[2], 1);
957         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
958         check_closed_broadcast!(nodes[2], true);
959
960         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
961         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
962         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
963
964         // Note that this checks that this is the only event on nodes[1], implying the
965         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
966         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager, [nodes[2].node.get_our_node_id()], 100000);
967
968         if use_intercept {
969                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
970                 // a intercept-doesn't-exist error.
971                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
972                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
973                 assert_eq!(forward_err, APIError::APIMisuseError {
974                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
975                 });
976         }
977
978         nodes[1].node.timer_tick_occurred();
979         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
980         assert_eq!(bs_commitment_tx.len(), 1);
981         check_added_monitors!(nodes[1], 1);
982
983         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
984         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
985
986         if use_cs_commitment {
987                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
988                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
989                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
990                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
991                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
992
993                 if claim_htlc {
994                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
995                 } else {
996                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
997                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
998                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
999                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
1000                 }
1001         } else {
1002                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
1003         }
1004
1005         if !claim_htlc {
1006                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1007         } else {
1008                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
1009         }
1010         check_added_monitors!(nodes[1], 1);
1011
1012         let events = nodes[1].node.get_and_clear_pending_msg_events();
1013         assert_eq!(events.len(), 1);
1014         match &events[0] {
1015                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1016                         if claim_htlc {
1017                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1018                         } else {
1019                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1020                         }
1021                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1022                 },
1023                 _ => panic!("Unexpected event"),
1024         }
1025
1026         if claim_htlc {
1027                 expect_payment_sent!(nodes[0], payment_preimage);
1028         } else {
1029                 expect_payment_failed!(nodes[0], payment_hash, false);
1030         }
1031 }
1032
1033 #[test]
1034 fn forwarded_payment_no_manager_persistence() {
1035         do_forwarded_payment_no_manager_persistence(true, true, false);
1036         do_forwarded_payment_no_manager_persistence(true, false, false);
1037         do_forwarded_payment_no_manager_persistence(false, false, false);
1038 }
1039
1040 #[test]
1041 fn intercepted_payment_no_manager_persistence() {
1042         do_forwarded_payment_no_manager_persistence(true, true, true);
1043         do_forwarded_payment_no_manager_persistence(true, false, true);
1044         do_forwarded_payment_no_manager_persistence(false, false, true);
1045 }
1046
1047 #[test]
1048 fn removed_payment_no_manager_persistence() {
1049         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1050         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1051         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1052         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1053         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1054         let chanmon_cfgs = create_chanmon_cfgs(3);
1055         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1056         let persister;
1057         let new_chain_monitor;
1058
1059         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1060         let nodes_1_deserialized;
1061
1062         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1063
1064         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1065         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1066
1067         let (_, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1068
1069         let node_encoded = nodes[1].node.encode();
1070
1071         nodes[2].node.fail_htlc_backwards(&payment_hash);
1072         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1073         check_added_monitors!(nodes[2], 1);
1074         let events = nodes[2].node.get_and_clear_pending_msg_events();
1075         assert_eq!(events.len(), 1);
1076         match &events[0] {
1077                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1078                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1079                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1080                 },
1081                 _ => panic!("Unexpected event"),
1082         }
1083
1084         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1085         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1086         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1087
1088         match nodes[1].node.pop_pending_event().unwrap() {
1089                 Event::ChannelClosed { ref reason, .. } => {
1090                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1091                 },
1092                 _ => panic!("Unexpected event"),
1093         }
1094
1095         nodes[1].node.test_process_background_events();
1096         check_added_monitors(&nodes[1], 1);
1097
1098         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1099         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1100         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1101         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1102         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1103
1104         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1105         check_added_monitors!(nodes[1], 1);
1106         let events = nodes[1].node.get_and_clear_pending_msg_events();
1107         assert_eq!(events.len(), 1);
1108         match &events[0] {
1109                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1110                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1111                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1112                 },
1113                 _ => panic!("Unexpected event"),
1114         }
1115
1116         expect_payment_failed!(nodes[0], payment_hash, false);
1117 }
1118
1119 #[test]
1120 fn test_reload_partial_funding_batch() {
1121         let chanmon_cfgs = create_chanmon_cfgs(3);
1122         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1123         let new_persister;
1124         let new_chain_monitor;
1125
1126         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1127         let new_channel_manager;
1128         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1129
1130         // Initiate channel opening and create the batch channel funding transaction.
1131         let (tx, funding_created_msgs) = create_batch_channel_funding(&nodes[0], &[
1132                 (&nodes[1], 100_000, 0, 42, None),
1133                 (&nodes[2], 200_000, 0, 43, None),
1134         ]);
1135
1136         // Go through the funding_created and funding_signed flow with node 1.
1137         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msgs[0]);
1138         check_added_monitors(&nodes[1], 1);
1139         expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
1140
1141         // The monitor is persisted when receiving funding_signed.
1142         let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
1143         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
1144         check_added_monitors(&nodes[0], 1);
1145
1146         // The transaction should not have been broadcast before all channels are ready.
1147         assert_eq!(nodes[0].tx_broadcaster.txn_broadcast().len(), 0);
1148
1149         // Reload the node while a subset of the channels in the funding batch have persisted monitors.
1150         let channel_id_1 = OutPoint { txid: tx.txid(), index: 0 }.to_channel_id();
1151         let node_encoded = nodes[0].node.encode();
1152         let channel_monitor_1_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1153         reload_node!(nodes[0], node_encoded, &[&channel_monitor_1_serialized], new_persister, new_chain_monitor, new_channel_manager);
1154
1155         // Process monitor events.
1156         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1157
1158         // The monitor should become closed.
1159         check_added_monitors(&nodes[0], 1);
1160         {
1161                 let mut monitor_updates = nodes[0].chain_monitor.monitor_updates.lock().unwrap();
1162                 let monitor_updates_1 = monitor_updates.get(&channel_id_1).unwrap();
1163                 assert_eq!(monitor_updates_1.len(), 1);
1164                 assert_eq!(monitor_updates_1[0].update_id, CLOSED_CHANNEL_UPDATE_ID);
1165         }
1166
1167         // The funding transaction should not have been broadcast, but we broadcast the force-close
1168         // transaction as part of closing the monitor.
1169         {
1170                 let broadcasted_txs = nodes[0].tx_broadcaster.txn_broadcast();
1171                 assert_eq!(broadcasted_txs.len(), 1);
1172                 assert!(broadcasted_txs[0].txid() != tx.txid());
1173                 assert_eq!(broadcasted_txs[0].input.len(), 1);
1174                 assert_eq!(broadcasted_txs[0].input[0].previous_output.txid, tx.txid());
1175         }
1176
1177         // Ensure the channels don't exist anymore.
1178         assert!(nodes[0].node.list_channels().is_empty());
1179 }