]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
Support failing blinded non-intro HTLCs after RAA processing.
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::{CLOSED_CHANNEL_UPDATE_ID, ChannelMonitor};
15 use crate::sign::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, Retry, RecipientOnionFields};
19 use crate::ln::msgs;
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::routing::router::{RouteParameters, PaymentParameters};
22 use crate::util::test_channel_signer::TestChannelSigner;
23 use crate::util::test_utils;
24 use crate::util::errors::APIError;
25 use crate::util::ser::{Writeable, ReadableArgs};
26 use crate::util::config::UserConfig;
27 use crate::util::string::UntrustedString;
28
29 use bitcoin::hash_types::BlockHash;
30
31 use crate::prelude::*;
32 use core::default::Default;
33 use crate::sync::Mutex;
34
35 use crate::ln::functional_test_utils::*;
36
37 #[test]
38 fn test_funding_peer_disconnect() {
39         // Test that we can lock in our funding tx while disconnected
40         let chanmon_cfgs = create_chanmon_cfgs(2);
41         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
42         let persister;
43         let new_chain_monitor;
44
45         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
46         let nodes_0_deserialized;
47         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
48         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
49
50         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
51         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
52
53         confirm_transaction(&nodes[0], &tx);
54         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
55         assert!(events_1.is_empty());
56
57         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
58         reconnect_args.send_channel_ready.1 = true;
59         reconnect_nodes(reconnect_args);
60
61         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
62         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
63
64         confirm_transaction(&nodes[1], &tx);
65         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
66         assert!(events_2.is_empty());
67
68         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
69                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
70         }, true).unwrap();
71         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
72         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
73                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
74         }, false).unwrap();
75         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
76
77         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
78         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
79         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
80         assert_eq!(events_3.len(), 1);
81         let as_channel_ready = match events_3[0] {
82                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
83                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
84                         msg.clone()
85                 },
86                 _ => panic!("Unexpected event {:?}", events_3[0]),
87         };
88
89         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
90         // announcement_signatures as well as channel_update.
91         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
92         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
93         assert_eq!(events_4.len(), 3);
94         let chan_id;
95         let bs_channel_ready = match events_4[0] {
96                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
97                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
98                         chan_id = msg.channel_id;
99                         msg.clone()
100                 },
101                 _ => panic!("Unexpected event {:?}", events_4[0]),
102         };
103         let bs_announcement_sigs = match events_4[1] {
104                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
105                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
106                         msg.clone()
107                 },
108                 _ => panic!("Unexpected event {:?}", events_4[1]),
109         };
110         match events_4[2] {
111                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
112                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
113                 },
114                 _ => panic!("Unexpected event {:?}", events_4[2]),
115         }
116
117         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
118         // generates a duplicative private channel_update
119         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
120         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
121         assert_eq!(events_5.len(), 1);
122         match events_5[0] {
123                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
124                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
125                 },
126                 _ => panic!("Unexpected event {:?}", events_5[0]),
127         };
128
129         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
130         // announcement_signatures.
131         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
132         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
133         assert_eq!(events_6.len(), 1);
134         let as_announcement_sigs = match events_6[0] {
135                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
136                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
137                         msg.clone()
138                 },
139                 _ => panic!("Unexpected event {:?}", events_6[0]),
140         };
141         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
142         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
143
144         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
145         // broadcast the channel announcement globally, as well as re-send its (now-public)
146         // channel_update.
147         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
148         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
149         assert_eq!(events_7.len(), 1);
150         let (chan_announcement, as_update) = match events_7[0] {
151                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
152                         (msg.clone(), update_msg.clone().unwrap())
153                 },
154                 _ => panic!("Unexpected event {:?}", events_7[0]),
155         };
156
157         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
158         // same channel_announcement.
159         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
160         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
161         assert_eq!(events_8.len(), 1);
162         let bs_update = match events_8[0] {
163                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
164                         assert_eq!(*msg, chan_announcement);
165                         update_msg.clone().unwrap()
166                 },
167                 _ => panic!("Unexpected event {:?}", events_8[0]),
168         };
169
170         // Provide the channel announcement and public updates to the network graph
171         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
172         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
173         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
174
175         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
176         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
177         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
178
179         // Check that after deserialization and reconnection we can still generate an identical
180         // channel_announcement from the cached signatures.
181         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
182
183         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
184
185         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
186
187         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
188 }
189
190 #[test]
191 fn test_no_txn_manager_serialize_deserialize() {
192         let chanmon_cfgs = create_chanmon_cfgs(2);
193         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
194         let persister;
195         let new_chain_monitor;
196
197         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
198         let nodes_0_deserialized;
199         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
200
201         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
202
203         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
204
205         let chan_0_monitor_serialized =
206                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
207         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
208
209         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
210                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
211         }, true).unwrap();
212         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
213         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
214                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
215         }, false).unwrap();
216         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
217
218         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
219         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
220         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
221         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
222
223         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
224         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
225         for node in nodes.iter() {
226                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
227                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
228                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
229         }
230
231         send_payment(&nodes[0], &[&nodes[1]], 1000000);
232 }
233
234 #[test]
235 fn test_manager_serialize_deserialize_events() {
236         // This test makes sure the events field in ChannelManager survives de/serialization
237         let chanmon_cfgs = create_chanmon_cfgs(2);
238         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
239         let persister;
240         let new_chain_monitor;
241
242         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
243         let nodes_0_deserialized;
244         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
245
246         // Start creating a channel, but stop right before broadcasting the funding transaction
247         let channel_value = 100000;
248         let push_msat = 10001;
249         let node_a = nodes.remove(0);
250         let node_b = nodes.remove(0);
251         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None, None).unwrap();
252         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
253         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
254
255         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
256
257         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
258         check_added_monitors!(node_a, 0);
259
260         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
261         {
262                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
263                 assert_eq!(added_monitors.len(), 1);
264                 assert_eq!(added_monitors[0].0, funding_output);
265                 added_monitors.clear();
266         }
267
268         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
269         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
270         {
271                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
272                 assert_eq!(added_monitors.len(), 1);
273                 assert_eq!(added_monitors[0].0, funding_output);
274                 added_monitors.clear();
275         }
276         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
277
278         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
279         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
280
281         nodes.push(node_a);
282         nodes.push(node_b);
283
284         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
285         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
286         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
287
288         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
289
290         // After deserializing, make sure the funding_transaction is still held by the channel manager
291         let events_4 = nodes[0].node.get_and_clear_pending_events();
292         assert_eq!(events_4.len(), 0);
293         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
294         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
295
296         // Make sure the channel is functioning as though the de/serialization never happened
297         assert_eq!(nodes[0].node.list_channels().len(), 1);
298
299         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
300                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
301         }, true).unwrap();
302         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
303         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
304                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
305         }, false).unwrap();
306         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
307
308         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
309         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
310         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
311         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
312
313         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
314         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
315         for node in nodes.iter() {
316                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
317                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
318                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
319         }
320
321         send_payment(&nodes[0], &[&nodes[1]], 1000000);
322 }
323
324 #[test]
325 fn test_simple_manager_serialize_deserialize() {
326         let chanmon_cfgs = create_chanmon_cfgs(2);
327         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
328         let persister;
329         let new_chain_monitor;
330
331         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
332         let nodes_0_deserialized;
333         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
334         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
335
336         let (our_payment_preimage, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
337         let (_, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
338
339         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
340
341         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
342         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
343
344         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
345
346         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
347         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
348 }
349
350 #[test]
351 fn test_manager_serialize_deserialize_inconsistent_monitor() {
352         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
353         let chanmon_cfgs = create_chanmon_cfgs(4);
354         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
355         let logger;
356         let fee_estimator;
357         let persister;
358         let new_chain_monitor;
359
360         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
361         let nodes_0_deserialized;
362         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
363
364         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
365         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
366         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
367
368         let mut node_0_stale_monitors_serialized = Vec::new();
369         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
370                 let mut writer = test_utils::TestVecWriter(Vec::new());
371                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
372                 node_0_stale_monitors_serialized.push(writer.0);
373         }
374
375         let (our_payment_preimage, ..) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
376
377         // Serialize the ChannelManager here, but the monitor we keep up-to-date
378         let nodes_0_serialized = nodes[0].node.encode();
379
380         route_payment(&nodes[0], &[&nodes[3]], 1000000);
381         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
382         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
383         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
384
385         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
386         // nodes[3])
387         let mut node_0_monitors_serialized = Vec::new();
388         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
389                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
390         }
391
392         logger = test_utils::TestLogger::new();
393         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
394         persister = test_utils::TestPersister::new();
395         let keys_manager = &chanmon_cfgs[0].keys_manager;
396         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
397         nodes[0].chain_monitor = &new_chain_monitor;
398
399
400         let mut node_0_stale_monitors = Vec::new();
401         for serialized in node_0_stale_monitors_serialized.iter() {
402                 let mut read = &serialized[..];
403                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
404                 assert!(read.is_empty());
405                 node_0_stale_monitors.push(monitor);
406         }
407
408         let mut node_0_monitors = Vec::new();
409         for serialized in node_0_monitors_serialized.iter() {
410                 let mut read = &serialized[..];
411                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
412                 assert!(read.is_empty());
413                 node_0_monitors.push(monitor);
414         }
415
416         let mut nodes_0_read = &nodes_0_serialized[..];
417         if let Err(msgs::DecodeError::InvalidValue) =
418                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
419                 default_config: UserConfig::default(),
420                 entropy_source: keys_manager,
421                 node_signer: keys_manager,
422                 signer_provider: keys_manager,
423                 fee_estimator: &fee_estimator,
424                 router: &nodes[0].router,
425                 chain_monitor: nodes[0].chain_monitor,
426                 tx_broadcaster: nodes[0].tx_broadcaster,
427                 logger: &logger,
428                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
429         }) { } else {
430                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
431         };
432
433         let mut nodes_0_read = &nodes_0_serialized[..];
434         let (_, nodes_0_deserialized_tmp) =
435                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
436                 default_config: UserConfig::default(),
437                 entropy_source: keys_manager,
438                 node_signer: keys_manager,
439                 signer_provider: keys_manager,
440                 fee_estimator: &fee_estimator,
441                 router: nodes[0].router,
442                 chain_monitor: nodes[0].chain_monitor,
443                 tx_broadcaster: nodes[0].tx_broadcaster,
444                 logger: &logger,
445                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
446         }).unwrap();
447         nodes_0_deserialized = nodes_0_deserialized_tmp;
448         assert!(nodes_0_read.is_empty());
449
450         for monitor in node_0_monitors.drain(..) {
451                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
452                         Ok(ChannelMonitorUpdateStatus::Completed));
453                 check_added_monitors!(nodes[0], 1);
454         }
455         nodes[0].node = &nodes_0_deserialized;
456
457         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[3].node.get_our_node_id()], 100000);
458         { // Channel close should result in a commitment tx
459                 nodes[0].node.timer_tick_occurred();
460                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
461                 assert_eq!(txn.len(), 1);
462                 check_spends!(txn[0], funding_tx);
463                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
464         }
465         check_added_monitors!(nodes[0], 1);
466
467         // nodes[1] and nodes[2] have no lost state with nodes[0]...
468         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
469         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
470         //... and we can even still claim the payment!
471         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
472
473         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
474                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
475         }, true).unwrap();
476         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
477         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
478                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
479         }, false).unwrap();
480         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
481         let mut found_err = false;
482         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
483                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
484                         match action {
485                                 &ErrorAction::SendErrorMessage { ref msg } => {
486                                         assert_eq!(msg.channel_id, channel_id);
487                                         assert!(!found_err);
488                                         found_err = true;
489                                 },
490                                 _ => panic!("Unexpected event!"),
491                         }
492                 }
493         }
494         assert!(found_err);
495 }
496
497 #[cfg(feature = "std")]
498 fn do_test_data_loss_protect(reconnect_panicing: bool, substantially_old: bool, not_stale: bool) {
499         // When we get a data_loss_protect proving we're behind, we immediately panic as the
500         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
501         // panic message informs the user they should force-close without broadcasting, which is tested
502         // if `reconnect_panicing` is not set.
503         let mut chanmon_cfgs = create_chanmon_cfgs(2);
504         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
505         // during signing due to revoked tx
506         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
507         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
508         let persister;
509         let new_chain_monitor;
510
511         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
512         let nodes_0_deserialized;
513
514         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
515
516         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
517
518         // Cache node A state before any channel update
519         let previous_node_state = nodes[0].node.encode();
520         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
521
522         assert!(!substantially_old || !not_stale, "substantially_old and not_stale doesn't make sense");
523         if not_stale || !substantially_old {
524                 // Previously, we'd only hit the data_loss_protect assertion if we had a state which
525                 // revoked at least two revocations ago, not the latest revocation. Here, we use
526                 // `not_stale` to test the boundary condition.
527                 let pay_params = PaymentParameters::for_keysend(nodes[1].node.get_our_node_id(), 100, false);
528                 let route_params = RouteParameters::from_payment_params_and_value(pay_params, 40000);
529                 nodes[0].node.send_spontaneous_payment_with_retry(None, RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]), route_params, Retry::Attempts(0)).unwrap();
530                 check_added_monitors(&nodes[0], 1);
531                 let update_add_commit = SendEvent::from_node(&nodes[0]);
532
533                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_commit.msgs[0]);
534                 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &update_add_commit.commitment_msg);
535                 check_added_monitors(&nodes[1], 1);
536                 let (raa, cs) = get_revoke_commit_msgs(&nodes[1], &nodes[0].node.get_our_node_id());
537
538                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa);
539                 check_added_monitors(&nodes[0], 1);
540                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
541                 if !not_stale {
542                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &cs);
543                         check_added_monitors(&nodes[0], 1);
544                         // A now revokes their original state, at which point reconnect should panic
545                         let raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
546                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &raa);
547                         check_added_monitors(&nodes[1], 1);
548                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
549                 }
550         } else {
551                 send_payment(&nodes[0], &[&nodes[1]], 8000000);
552                 send_payment(&nodes[0], &[&nodes[1]], 8000000);
553         }
554
555         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
556         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
557
558         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
559
560         if reconnect_panicing {
561                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
562                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
563                 }, true).unwrap();
564                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
565                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
566                 }, false).unwrap();
567
568                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
569
570                 // If A has fallen behind substantially, B should send it a message letting it know
571                 // that.
572                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
573                 let reestablish_msg;
574                 if substantially_old {
575                         let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction: 0 (received) vs 4 (expected)".to_owned();
576
577                         let warn_reestablish = nodes[1].node.get_and_clear_pending_msg_events();
578                         assert_eq!(warn_reestablish.len(), 2);
579                         match warn_reestablish[1] {
580                                 MessageSendEvent::HandleError { action: ErrorAction::SendWarningMessage { ref msg, .. }, .. } => {
581                                         assert_eq!(msg.data, warn_msg);
582                                 },
583                                 _ => panic!("Unexpected events: {:?}", warn_reestablish),
584                         }
585                         reestablish_msg = match &warn_reestablish[0] {
586                                 MessageSendEvent::SendChannelReestablish { msg, .. } => msg.clone(),
587                                 _ => panic!("Unexpected events: {:?}", warn_reestablish),
588                         };
589                 } else {
590                         let msgs = nodes[1].node.get_and_clear_pending_msg_events();
591                         assert!(msgs.len() >= 4);
592                         match msgs.last() {
593                                 Some(MessageSendEvent::SendChannelUpdate { .. }) => {},
594                                 _ => panic!("Unexpected events: {:?}", msgs),
595                         }
596                         assert!(msgs.iter().any(|msg| matches!(msg, MessageSendEvent::SendRevokeAndACK { .. })));
597                         assert!(msgs.iter().any(|msg| matches!(msg, MessageSendEvent::UpdateHTLCs { .. })));
598                         reestablish_msg = match &msgs[0] {
599                                 MessageSendEvent::SendChannelReestablish { msg, .. } => msg.clone(),
600                                 _ => panic!("Unexpected events: {:?}", msgs),
601                         };
602                 }
603
604                 {
605                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
606                         // The node B should never force-close the channel.
607                         assert!(node_txn.is_empty());
608                 }
609
610                 // Check A panics upon seeing proof it has fallen behind.
611                 let reconnect_res = std::panic::catch_unwind(|| {
612                         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_msg);
613                 });
614                 if not_stale {
615                         assert!(reconnect_res.is_ok());
616                         // At this point A gets confused because B expects a commitment state newer than A
617                         // has sent, but not a newer revocation secret, so A just (correctly) closes.
618                         check_closed_broadcast(&nodes[0], 1, true);
619                         check_added_monitors(&nodes[0], 1);
620                         check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError {
621                                 err: "Peer attempted to reestablish channel with a future remote commitment transaction: 2 (received) vs 1 (expected)".to_owned()
622                         }, [nodes[1].node.get_our_node_id()], 1000000);
623                 } else {
624                         assert!(reconnect_res.is_err());
625                         // Skip the `Drop` handler for `Node`s as some may be in an invalid (panicked) state.
626                         std::mem::forget(nodes);
627                 }
628         } else {
629                 assert!(!not_stale, "We only care about the stale case when not testing panicking");
630
631                 nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
632                 check_added_monitors!(nodes[0], 1);
633                 check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 1000000);
634                 {
635                         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
636                         assert_eq!(node_txn.len(), 0);
637                 }
638
639                 for msg in nodes[0].node.get_and_clear_pending_msg_events() {
640                         if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
641                         } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
642                                 match action {
643                                         &ErrorAction::DisconnectPeer { ref msg } => {
644                                                 assert_eq!(msg.as_ref().unwrap().data, "Channel force-closed");
645                                         },
646                                         _ => panic!("Unexpected event!"),
647                                 }
648                         } else {
649                                 panic!("Unexpected event {:?}", msg)
650                         }
651                 }
652
653                 // after the warning message sent by B, we should not able to
654                 // use the channel, or reconnect with success to the channel.
655                 assert!(nodes[0].node.list_usable_channels().is_empty());
656                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
657                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
658                 }, true).unwrap();
659                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
660                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
661                 }, false).unwrap();
662                 let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
663
664                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
665                 let mut err_msgs_0 = Vec::with_capacity(1);
666                 if let MessageSendEvent::HandleError { ref action, .. } = nodes[0].node.get_and_clear_pending_msg_events()[1] {
667                         match action {
668                                 &ErrorAction::SendErrorMessage { ref msg } => {
669                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
670                                         err_msgs_0.push(msg.clone());
671                                 },
672                                 _ => panic!("Unexpected event!"),
673                         }
674                 } else {
675                         panic!("Unexpected event!");
676                 }
677                 assert_eq!(err_msgs_0.len(), 1);
678                 nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
679                 assert!(nodes[1].node.list_usable_channels().is_empty());
680                 check_added_monitors!(nodes[1], 1);
681                 check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
682                         , [nodes[0].node.get_our_node_id()], 1000000);
683                 check_closed_broadcast!(nodes[1], false);
684         }
685 }
686
687 #[test]
688 #[cfg(feature = "std")]
689 fn test_data_loss_protect() {
690         do_test_data_loss_protect(true, false, true);
691         do_test_data_loss_protect(true, true, false);
692         do_test_data_loss_protect(true, false, false);
693         do_test_data_loss_protect(false, true, false);
694         do_test_data_loss_protect(false, false, false);
695 }
696
697 #[test]
698 fn test_forwardable_regen() {
699         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
700         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
701         // HTLCs.
702         // We test it for both payment receipt and payment forwarding.
703
704         let chanmon_cfgs = create_chanmon_cfgs(3);
705         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
706         let persister;
707         let new_chain_monitor;
708         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
709         let nodes_1_deserialized;
710         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
711         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
712         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
713
714         // First send a payment to nodes[1]
715         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
716         nodes[0].node.send_payment_with_route(&route, payment_hash,
717                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
718         check_added_monitors!(nodes[0], 1);
719
720         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
721         assert_eq!(events.len(), 1);
722         let payment_event = SendEvent::from_event(events.pop().unwrap());
723         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
724         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
725
726         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
727
728         // Next send a payment which is forwarded by nodes[1]
729         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
730         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
731                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
732         check_added_monitors!(nodes[0], 1);
733
734         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
735         assert_eq!(events.len(), 1);
736         let payment_event = SendEvent::from_event(events.pop().unwrap());
737         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
738         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
739
740         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
741         // generated
742         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
743
744         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
745         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
746         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
747
748         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
749         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
750         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
751
752         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
753         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
754         // the commitment state.
755         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
756         reconnect_args.send_channel_ready = (true, true);
757         reconnect_nodes(reconnect_args);
758
759         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
760
761         expect_pending_htlcs_forwardable!(nodes[1]);
762         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
763         check_added_monitors!(nodes[1], 1);
764
765         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
766         assert_eq!(events.len(), 1);
767         let payment_event = SendEvent::from_event(events.pop().unwrap());
768         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
769         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
770         expect_pending_htlcs_forwardable!(nodes[2]);
771         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
772
773         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
774         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
775 }
776
777 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
778         // Test what happens if a node receives an MPP payment, claims it, but crashes before
779         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
780         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
781         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
782         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
783         // not have the preimage tied to the still-pending HTLC.
784         //
785         // To get to the correct state, on startup we should propagate the preimage to the
786         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
787         // receiving the preimage without a state update.
788         //
789         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
790         // definitely claimed.
791         let chanmon_cfgs = create_chanmon_cfgs(4);
792         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
793         let persister;
794         let new_chain_monitor;
795
796         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
797         let nodes_3_deserialized;
798
799         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
800
801         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
802         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
803         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
804         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
805
806         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
807         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
808         assert_eq!(route.paths.len(), 2);
809         route.paths.sort_by(|path_a, _| {
810                 // Sort the path so that the path through nodes[1] comes first
811                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
812                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
813         });
814
815         nodes[0].node.send_payment_with_route(&route, payment_hash,
816                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
817         check_added_monitors!(nodes[0], 2);
818
819         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
820         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
821         assert_eq!(send_events.len(), 2);
822         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
823         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
824         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None, false);
825         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None, false);
826
827         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
828         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
829         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
830         if !persist_both_monitors {
831                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
832                         if outpoint.to_channel_id() == chan_id_not_persisted {
833                                 assert!(original_monitor.0.is_empty());
834                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
835                         }
836                 }
837         }
838
839         let original_manager = nodes[3].node.encode();
840
841         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
842
843         nodes[3].node.claim_funds(payment_preimage);
844         check_added_monitors!(nodes[3], 2);
845         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
846
847         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
848         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
849         // with the old ChannelManager.
850         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
851         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
852                 if outpoint.to_channel_id() == chan_id_persisted {
853                         assert!(updated_monitor.0.is_empty());
854                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
855                 }
856         }
857         // If `persist_both_monitors` is set, get the second monitor here as well
858         if persist_both_monitors {
859                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
860                         if outpoint.to_channel_id() == chan_id_not_persisted {
861                                 assert!(original_monitor.0.is_empty());
862                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
863                         }
864                 }
865         }
866
867         // Now restart nodes[3].
868         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
869
870         // On startup the preimage should have been copied into the non-persisted monitor:
871         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
872         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
873
874         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
875         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
876
877         // During deserialization, we should have closed one channel and broadcast its latest
878         // commitment transaction. We should also still have the original PaymentClaimable event we
879         // never finished processing.
880         let events = nodes[3].node.get_and_clear_pending_events();
881         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
882         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
883         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
884         if persist_both_monitors {
885                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
886                 check_added_monitors(&nodes[3], 2);
887         } else {
888                 check_added_monitors(&nodes[3], 1);
889         }
890
891         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
892         // ChannelManager prior to handling the original one.
893         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
894                 events[if persist_both_monitors { 3 } else { 2 }]
895         {
896                 assert_eq!(payment_hash, our_payment_hash);
897         } else { panic!(); }
898
899         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
900         if !persist_both_monitors {
901                 // If one of the two channels is still live, reveal the payment preimage over it.
902
903                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
904                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
905                 }, true).unwrap();
906                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
907                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
908                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
909                 }, false).unwrap();
910                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
911
912                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
913                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
914                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
915
916                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
917
918                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
919                 // claim should fly.
920                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
921                 check_added_monitors!(nodes[3], 1);
922                 assert_eq!(ds_msgs.len(), 2);
923                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
924
925                 let cs_updates = match ds_msgs[1] {
926                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
927                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
928                                 check_added_monitors!(nodes[2], 1);
929                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
930                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
931                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
932                                 cs_updates
933                         }
934                         _ => panic!(),
935                 };
936
937                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
938                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
939                 expect_payment_sent!(nodes[0], payment_preimage);
940         }
941 }
942
943 #[test]
944 fn test_partial_claim_before_restart() {
945         do_test_partial_claim_before_restart(false);
946         do_test_partial_claim_before_restart(true);
947 }
948
949 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
950         if !use_cs_commitment { assert!(!claim_htlc); }
951         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
952         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
953         // it back until the ChannelMonitor decides the fate of the HTLC.
954         // This was never an issue, but it may be easy to regress here going forward.
955         let chanmon_cfgs = create_chanmon_cfgs(3);
956         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
957         let persister;
958         let new_chain_monitor;
959
960         let mut intercept_forwards_config = test_default_channel_config();
961         intercept_forwards_config.accept_intercept_htlcs = true;
962         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
963         let nodes_1_deserialized;
964
965         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
966
967         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
968         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
969
970         let intercept_scid = nodes[1].node.get_intercept_scid();
971
972         let (mut route, payment_hash, payment_preimage, payment_secret) =
973                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
974         if use_intercept {
975                 route.paths[0].hops[1].short_channel_id = intercept_scid;
976         }
977         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
978         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
979         nodes[0].node.send_payment_with_route(&route, payment_hash,
980                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
981         check_added_monitors!(nodes[0], 1);
982
983         let payment_event = SendEvent::from_node(&nodes[0]);
984         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
985         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
986
987         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
988         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
989         // present when we serialized.
990         let node_encoded = nodes[1].node.encode();
991
992         let mut intercept_id = None;
993         let mut expected_outbound_amount_msat = None;
994         if use_intercept {
995                 let events = nodes[1].node.get_and_clear_pending_events();
996                 assert_eq!(events.len(), 1);
997                 match events[0] {
998                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
999                                 intercept_id = Some(ev_id);
1000                                 expected_outbound_amount_msat = Some(ev_amt);
1001                         },
1002                         _ => panic!()
1003                 }
1004                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
1005                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
1006         }
1007
1008         expect_pending_htlcs_forwardable!(nodes[1]);
1009
1010         let payment_event = SendEvent::from_node(&nodes[1]);
1011         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1012         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
1013         check_added_monitors!(nodes[2], 1);
1014
1015         if claim_htlc {
1016                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
1017                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
1018         }
1019         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
1020
1021         let _ = nodes[2].node.get_and_clear_pending_msg_events();
1022
1023         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
1024         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1025         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
1026
1027         check_added_monitors!(nodes[2], 1);
1028         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
1029         check_closed_broadcast!(nodes[2], true);
1030
1031         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1032         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1033         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1034
1035         // Note that this checks that this is the only event on nodes[1], implying the
1036         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
1037         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager, [nodes[2].node.get_our_node_id()], 100000);
1038
1039         if use_intercept {
1040                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
1041                 // a intercept-doesn't-exist error.
1042                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
1043                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
1044                 assert_eq!(forward_err, APIError::APIMisuseError {
1045                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
1046                 });
1047         }
1048
1049         nodes[1].node.timer_tick_occurred();
1050         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1051         assert_eq!(bs_commitment_tx.len(), 1);
1052         check_added_monitors!(nodes[1], 1);
1053
1054         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1055         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1056
1057         if use_cs_commitment {
1058                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
1059                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
1060                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
1061                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1062                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1063
1064                 if claim_htlc {
1065                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
1066                 } else {
1067                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
1068                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1069                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
1070                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
1071                 }
1072         } else {
1073                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
1074         }
1075
1076         if !claim_htlc {
1077                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1078         } else {
1079                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
1080         }
1081         check_added_monitors!(nodes[1], 1);
1082
1083         let events = nodes[1].node.get_and_clear_pending_msg_events();
1084         assert_eq!(events.len(), 1);
1085         match &events[0] {
1086                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1087                         if claim_htlc {
1088                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1089                         } else {
1090                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1091                         }
1092                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1093                 },
1094                 _ => panic!("Unexpected event"),
1095         }
1096
1097         if claim_htlc {
1098                 expect_payment_sent!(nodes[0], payment_preimage);
1099         } else {
1100                 expect_payment_failed!(nodes[0], payment_hash, false);
1101         }
1102 }
1103
1104 #[test]
1105 fn forwarded_payment_no_manager_persistence() {
1106         do_forwarded_payment_no_manager_persistence(true, true, false);
1107         do_forwarded_payment_no_manager_persistence(true, false, false);
1108         do_forwarded_payment_no_manager_persistence(false, false, false);
1109 }
1110
1111 #[test]
1112 fn intercepted_payment_no_manager_persistence() {
1113         do_forwarded_payment_no_manager_persistence(true, true, true);
1114         do_forwarded_payment_no_manager_persistence(true, false, true);
1115         do_forwarded_payment_no_manager_persistence(false, false, true);
1116 }
1117
1118 #[test]
1119 fn removed_payment_no_manager_persistence() {
1120         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1121         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1122         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1123         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1124         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1125         let chanmon_cfgs = create_chanmon_cfgs(3);
1126         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1127         let persister;
1128         let new_chain_monitor;
1129
1130         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1131         let nodes_1_deserialized;
1132
1133         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1134
1135         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1136         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1137
1138         let (_, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1139
1140         let node_encoded = nodes[1].node.encode();
1141
1142         nodes[2].node.fail_htlc_backwards(&payment_hash);
1143         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1144         check_added_monitors!(nodes[2], 1);
1145         let events = nodes[2].node.get_and_clear_pending_msg_events();
1146         assert_eq!(events.len(), 1);
1147         match &events[0] {
1148                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1149                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1150                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1151                 },
1152                 _ => panic!("Unexpected event"),
1153         }
1154
1155         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1156         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1157         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1158
1159         match nodes[1].node.pop_pending_event().unwrap() {
1160                 Event::ChannelClosed { ref reason, .. } => {
1161                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1162                 },
1163                 _ => panic!("Unexpected event"),
1164         }
1165
1166         nodes[1].node.test_process_background_events();
1167         check_added_monitors(&nodes[1], 1);
1168
1169         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1170         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1171         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1172         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1173         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1174
1175         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1176         check_added_monitors!(nodes[1], 1);
1177         let events = nodes[1].node.get_and_clear_pending_msg_events();
1178         assert_eq!(events.len(), 1);
1179         match &events[0] {
1180                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1181                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1182                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1183                 },
1184                 _ => panic!("Unexpected event"),
1185         }
1186
1187         expect_payment_failed!(nodes[0], payment_hash, false);
1188 }
1189
1190 #[test]
1191 fn test_reload_partial_funding_batch() {
1192         let chanmon_cfgs = create_chanmon_cfgs(3);
1193         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1194         let new_persister;
1195         let new_chain_monitor;
1196
1197         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1198         let new_channel_manager;
1199         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1200
1201         // Initiate channel opening and create the batch channel funding transaction.
1202         let (tx, funding_created_msgs) = create_batch_channel_funding(&nodes[0], &[
1203                 (&nodes[1], 100_000, 0, 42, None),
1204                 (&nodes[2], 200_000, 0, 43, None),
1205         ]);
1206
1207         // Go through the funding_created and funding_signed flow with node 1.
1208         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msgs[0]);
1209         check_added_monitors(&nodes[1], 1);
1210         expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
1211
1212         // The monitor is persisted when receiving funding_signed.
1213         let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
1214         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
1215         check_added_monitors(&nodes[0], 1);
1216
1217         // The transaction should not have been broadcast before all channels are ready.
1218         assert_eq!(nodes[0].tx_broadcaster.txn_broadcast().len(), 0);
1219
1220         // Reload the node while a subset of the channels in the funding batch have persisted monitors.
1221         let channel_id_1 = OutPoint { txid: tx.txid(), index: 0 }.to_channel_id();
1222         let node_encoded = nodes[0].node.encode();
1223         let channel_monitor_1_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1224         reload_node!(nodes[0], node_encoded, &[&channel_monitor_1_serialized], new_persister, new_chain_monitor, new_channel_manager);
1225
1226         // Process monitor events.
1227         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1228
1229         // The monitor should become closed.
1230         check_added_monitors(&nodes[0], 1);
1231         {
1232                 let mut monitor_updates = nodes[0].chain_monitor.monitor_updates.lock().unwrap();
1233                 let monitor_updates_1 = monitor_updates.get(&channel_id_1).unwrap();
1234                 assert_eq!(monitor_updates_1.len(), 1);
1235                 assert_eq!(monitor_updates_1[0].update_id, CLOSED_CHANNEL_UPDATE_ID);
1236         }
1237
1238         // The funding transaction should not have been broadcast, but we broadcast the force-close
1239         // transaction as part of closing the monitor.
1240         {
1241                 let broadcasted_txs = nodes[0].tx_broadcaster.txn_broadcast();
1242                 assert_eq!(broadcasted_txs.len(), 1);
1243                 assert!(broadcasted_txs[0].txid() != tx.txid());
1244                 assert_eq!(broadcasted_txs[0].input.len(), 1);
1245                 assert_eq!(broadcasted_txs[0].input[0].previous_output.txid, tx.txid());
1246         }
1247
1248         // Ensure the channels don't exist anymore.
1249         assert!(nodes[0].node.list_channels().is_empty());
1250 }