]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
6643d34ca17005e8bdeb62a6a52a27f287245903
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::ChannelMonitor;
15 use crate::chain::keysinterface::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId};
18 use crate::ln::msgs;
19 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
20 use crate::util::enforcing_trait_impls::EnforcingSigner;
21 use crate::util::test_utils;
22 use crate::util::errors::APIError;
23 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26
27 use bitcoin::hash_types::BlockHash;
28
29 use crate::prelude::*;
30 use core::default::Default;
31 use crate::sync::Mutex;
32
33 use crate::ln::functional_test_utils::*;
34
35 #[test]
36 fn test_funding_peer_disconnect() {
37         // Test that we can lock in our funding tx while disconnected
38         let chanmon_cfgs = create_chanmon_cfgs(2);
39         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
40         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
41         let persister: test_utils::TestPersister;
42         let new_chain_monitor: test_utils::TestChainMonitor;
43         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
44         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
45         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
46
47         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
48         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
49
50         confirm_transaction(&nodes[0], &tx);
51         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
52         assert!(events_1.is_empty());
53
54         reconnect_nodes(&nodes[0], &nodes[1], (false, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
55
56         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
57         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
58
59         confirm_transaction(&nodes[1], &tx);
60         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
61         assert!(events_2.is_empty());
62
63         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
64         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
65         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
66         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
67
68         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
69         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
70         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
71         assert_eq!(events_3.len(), 1);
72         let as_channel_ready = match events_3[0] {
73                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
74                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
75                         msg.clone()
76                 },
77                 _ => panic!("Unexpected event {:?}", events_3[0]),
78         };
79
80         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
81         // announcement_signatures as well as channel_update.
82         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
83         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
84         assert_eq!(events_4.len(), 3);
85         let chan_id;
86         let bs_channel_ready = match events_4[0] {
87                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
88                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
89                         chan_id = msg.channel_id;
90                         msg.clone()
91                 },
92                 _ => panic!("Unexpected event {:?}", events_4[0]),
93         };
94         let bs_announcement_sigs = match events_4[1] {
95                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
96                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
97                         msg.clone()
98                 },
99                 _ => panic!("Unexpected event {:?}", events_4[1]),
100         };
101         match events_4[2] {
102                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
103                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
104                 },
105                 _ => panic!("Unexpected event {:?}", events_4[2]),
106         }
107
108         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
109         // generates a duplicative private channel_update
110         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
111         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
112         assert_eq!(events_5.len(), 1);
113         match events_5[0] {
114                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
115                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
116                 },
117                 _ => panic!("Unexpected event {:?}", events_5[0]),
118         };
119
120         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
121         // announcement_signatures.
122         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
123         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
124         assert_eq!(events_6.len(), 1);
125         let as_announcement_sigs = match events_6[0] {
126                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
127                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
128                         msg.clone()
129                 },
130                 _ => panic!("Unexpected event {:?}", events_6[0]),
131         };
132         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
133         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
134
135         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
136         // broadcast the channel announcement globally, as well as re-send its (now-public)
137         // channel_update.
138         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
139         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
140         assert_eq!(events_7.len(), 1);
141         let (chan_announcement, as_update) = match events_7[0] {
142                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
143                         (msg.clone(), update_msg.clone().unwrap())
144                 },
145                 _ => panic!("Unexpected event {:?}", events_7[0]),
146         };
147
148         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
149         // same channel_announcement.
150         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
151         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
152         assert_eq!(events_8.len(), 1);
153         let bs_update = match events_8[0] {
154                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
155                         assert_eq!(*msg, chan_announcement);
156                         update_msg.clone().unwrap()
157                 },
158                 _ => panic!("Unexpected event {:?}", events_8[0]),
159         };
160
161         // Provide the channel announcement and public updates to the network graph
162         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
163         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
164         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
165
166         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
167         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
168         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
169
170         // Check that after deserialization and reconnection we can still generate an identical
171         // channel_announcement from the cached signatures.
172         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
173
174         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
175
176         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
177
178         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
179 }
180
181 #[test]
182 fn test_no_txn_manager_serialize_deserialize() {
183         let chanmon_cfgs = create_chanmon_cfgs(2);
184         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
185         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
186         let persister: test_utils::TestPersister;
187         let new_chain_monitor: test_utils::TestChainMonitor;
188         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
189         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
190
191         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
192
193         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
194
195         let chan_0_monitor_serialized =
196                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
197         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
198
199         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
200         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
201         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
202         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
203
204         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
205         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
206         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
207         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
208
209         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
210         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
211         for node in nodes.iter() {
212                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
213                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
214                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
215         }
216
217         send_payment(&nodes[0], &[&nodes[1]], 1000000);
218 }
219
220 #[test]
221 fn test_manager_serialize_deserialize_events() {
222         // This test makes sure the events field in ChannelManager survives de/serialization
223         let chanmon_cfgs = create_chanmon_cfgs(2);
224         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
225         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
226         let persister: test_utils::TestPersister;
227         let new_chain_monitor: test_utils::TestChainMonitor;
228         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
229         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
230
231         // Start creating a channel, but stop right before broadcasting the funding transaction
232         let channel_value = 100000;
233         let push_msat = 10001;
234         let node_a = nodes.remove(0);
235         let node_b = nodes.remove(0);
236         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
237         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
238         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
239
240         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
241
242         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
243         check_added_monitors!(node_a, 0);
244
245         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
246         {
247                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
248                 assert_eq!(added_monitors.len(), 1);
249                 assert_eq!(added_monitors[0].0, funding_output);
250                 added_monitors.clear();
251         }
252
253         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
254         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
255         {
256                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
257                 assert_eq!(added_monitors.len(), 1);
258                 assert_eq!(added_monitors[0].0, funding_output);
259                 added_monitors.clear();
260         }
261         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
262
263         nodes.push(node_a);
264         nodes.push(node_b);
265
266         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
267         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
268         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
269
270         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
271
272         // After deserializing, make sure the funding_transaction is still held by the channel manager
273         let events_4 = nodes[0].node.get_and_clear_pending_events();
274         assert_eq!(events_4.len(), 0);
275         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
276         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
277
278         // Make sure the channel is functioning as though the de/serialization never happened
279         assert_eq!(nodes[0].node.list_channels().len(), 1);
280
281         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
282         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
283         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
284         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
285
286         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
287         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
288         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
289         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
290
291         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
292         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
293         for node in nodes.iter() {
294                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
295                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
296                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
297         }
298
299         send_payment(&nodes[0], &[&nodes[1]], 1000000);
300 }
301
302 #[test]
303 fn test_simple_manager_serialize_deserialize() {
304         let chanmon_cfgs = create_chanmon_cfgs(2);
305         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
306         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
307         let persister: test_utils::TestPersister;
308         let new_chain_monitor: test_utils::TestChainMonitor;
309         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
310         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
311         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
312
313         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
314         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
315
316         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
317
318         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
319         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
320
321         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
322
323         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
324         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
325 }
326
327 #[test]
328 fn test_manager_serialize_deserialize_inconsistent_monitor() {
329         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
330         let chanmon_cfgs = create_chanmon_cfgs(4);
331         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
332         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
333         let logger: test_utils::TestLogger;
334         let fee_estimator: test_utils::TestFeeEstimator;
335         let persister: test_utils::TestPersister;
336         let new_chain_monitor: test_utils::TestChainMonitor;
337         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
338         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
339         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
340         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
341         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
342
343         let mut node_0_stale_monitors_serialized = Vec::new();
344         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
345                 let mut writer = test_utils::TestVecWriter(Vec::new());
346                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
347                 node_0_stale_monitors_serialized.push(writer.0);
348         }
349
350         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
351
352         // Serialize the ChannelManager here, but the monitor we keep up-to-date
353         let nodes_0_serialized = nodes[0].node.encode();
354
355         route_payment(&nodes[0], &[&nodes[3]], 1000000);
356         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
357         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
358         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
359
360         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
361         // nodes[3])
362         let mut node_0_monitors_serialized = Vec::new();
363         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
364                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
365         }
366
367         logger = test_utils::TestLogger::new();
368         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
369         persister = test_utils::TestPersister::new();
370         let keys_manager = &chanmon_cfgs[0].keys_manager;
371         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster.clone(), &logger, &fee_estimator, &persister, keys_manager);
372         nodes[0].chain_monitor = &new_chain_monitor;
373
374
375         let mut node_0_stale_monitors = Vec::new();
376         for serialized in node_0_stale_monitors_serialized.iter() {
377                 let mut read = &serialized[..];
378                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
379                 assert!(read.is_empty());
380                 node_0_stale_monitors.push(monitor);
381         }
382
383         let mut node_0_monitors = Vec::new();
384         for serialized in node_0_monitors_serialized.iter() {
385                 let mut read = &serialized[..];
386                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
387                 assert!(read.is_empty());
388                 node_0_monitors.push(monitor);
389         }
390
391         let mut nodes_0_read = &nodes_0_serialized[..];
392         if let Err(msgs::DecodeError::InvalidValue) =
393                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
394                 default_config: UserConfig::default(),
395                 entropy_source: keys_manager,
396                 node_signer: keys_manager,
397                 signer_provider: keys_manager,
398                 fee_estimator: &fee_estimator,
399                 router: &nodes[0].router,
400                 chain_monitor: nodes[0].chain_monitor,
401                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
402                 logger: &logger,
403                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
404         }) { } else {
405                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
406         };
407
408         let mut nodes_0_read = &nodes_0_serialized[..];
409         let (_, nodes_0_deserialized_tmp) =
410                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
411                 default_config: UserConfig::default(),
412                 entropy_source: keys_manager,
413                 node_signer: keys_manager,
414                 signer_provider: keys_manager,
415                 fee_estimator: &fee_estimator,
416                 router: nodes[0].router,
417                 chain_monitor: nodes[0].chain_monitor,
418                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
419                 logger: &logger,
420                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
421         }).unwrap();
422         nodes_0_deserialized = nodes_0_deserialized_tmp;
423         assert!(nodes_0_read.is_empty());
424
425         { // Channel close should result in a commitment tx
426                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
427                 assert_eq!(txn.len(), 1);
428                 check_spends!(txn[0], funding_tx);
429                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
430         }
431
432         for monitor in node_0_monitors.drain(..) {
433                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
434                         ChannelMonitorUpdateStatus::Completed);
435                 check_added_monitors!(nodes[0], 1);
436         }
437         nodes[0].node = &nodes_0_deserialized;
438         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
439
440         // nodes[1] and nodes[2] have no lost state with nodes[0]...
441         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
442         reconnect_nodes(&nodes[0], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
443         //... and we can even still claim the payment!
444         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
445
446         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
447         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
448         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: nodes[3].node.init_features(), remote_network_address: None }).unwrap();
449         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
450         let mut found_err = false;
451         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
452                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
453                         match action {
454                                 &ErrorAction::SendErrorMessage { ref msg } => {
455                                         assert_eq!(msg.channel_id, channel_id);
456                                         assert!(!found_err);
457                                         found_err = true;
458                                 },
459                                 _ => panic!("Unexpected event!"),
460                         }
461                 }
462         }
463         assert!(found_err);
464 }
465
466 fn do_test_data_loss_protect(reconnect_panicing: bool) {
467         // When we get a data_loss_protect proving we're behind, we immediately panic as the
468         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
469         // panic message informs the user they should force-close without broadcasting, which is tested
470         // if `reconnect_panicing` is not set.
471         let mut chanmon_cfgs = create_chanmon_cfgs(2);
472         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
473         // during signing due to revoked tx
474         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
475         let persister;
476         let new_chain_monitor;
477         let nodes_0_deserialized;
478         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
479         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
480         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
481
482         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
483
484         // Cache node A state before any channel update
485         let previous_node_state = nodes[0].node.encode();
486         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
487
488         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
489         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
490
491         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
492         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
493
494         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
495
496         if reconnect_panicing {
497                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
498                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
499
500                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
501
502                 // Check we close channel detecting A is fallen-behind
503                 // Check that we sent the warning message when we detected that A has fallen behind,
504                 // and give the possibility for A to recover from the warning.
505                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
506                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
507                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
508
509                 {
510                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
511                         // The node B should not broadcast the transaction to force close the channel!
512                         assert!(node_txn.is_empty());
513                 }
514
515                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
516                 // Check A panics upon seeing proof it has fallen behind.
517                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
518                 return; // By this point we should have panic'ed!
519         }
520
521         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
522         check_added_monitors!(nodes[0], 1);
523         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
524         {
525                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
526                 assert_eq!(node_txn.len(), 0);
527         }
528
529         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
530                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
531                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
532                         match action {
533                                 &ErrorAction::SendErrorMessage { ref msg } => {
534                                         assert_eq!(msg.data, "Channel force-closed");
535                                 },
536                                 _ => panic!("Unexpected event!"),
537                         }
538                 } else {
539                         panic!("Unexpected event {:?}", msg)
540                 }
541         }
542
543         // after the warning message sent by B, we should not able to
544         // use the channel, or reconnect with success to the channel.
545         assert!(nodes[0].node.list_usable_channels().is_empty());
546         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }).unwrap();
547         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }).unwrap();
548         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
549
550         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
551         let mut err_msgs_0 = Vec::with_capacity(1);
552         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
553                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
554                         match action {
555                                 &ErrorAction::SendErrorMessage { ref msg } => {
556                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
557                                         err_msgs_0.push(msg.clone());
558                                 },
559                                 _ => panic!("Unexpected event!"),
560                         }
561                 } else {
562                         panic!("Unexpected event!");
563                 }
564         }
565         assert_eq!(err_msgs_0.len(), 1);
566         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
567         assert!(nodes[1].node.list_usable_channels().is_empty());
568         check_added_monitors!(nodes[1], 1);
569         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
570         check_closed_broadcast!(nodes[1], false);
571 }
572
573 #[test]
574 #[should_panic]
575 fn test_data_loss_protect_showing_stale_state_panics() {
576         do_test_data_loss_protect(true);
577 }
578
579 #[test]
580 fn test_force_close_without_broadcast() {
581         do_test_data_loss_protect(false);
582 }
583
584 #[test]
585 fn test_forwardable_regen() {
586         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
587         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
588         // HTLCs.
589         // We test it for both payment receipt and payment forwarding.
590
591         let chanmon_cfgs = create_chanmon_cfgs(3);
592         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
593         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
594         let persister: test_utils::TestPersister;
595         let new_chain_monitor: test_utils::TestChainMonitor;
596         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
597         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
598         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
599         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
600
601         // First send a payment to nodes[1]
602         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
603         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
604         check_added_monitors!(nodes[0], 1);
605
606         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
607         assert_eq!(events.len(), 1);
608         let payment_event = SendEvent::from_event(events.pop().unwrap());
609         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
610         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
611
612         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
613
614         // Next send a payment which is forwarded by nodes[1]
615         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
616         nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
617         check_added_monitors!(nodes[0], 1);
618
619         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
620         assert_eq!(events.len(), 1);
621         let payment_event = SendEvent::from_event(events.pop().unwrap());
622         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
623         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
624
625         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
626         // generated
627         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
628
629         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
630         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
631         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
632
633         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
634         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
635         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
636
637         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
638         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
639         // the commitment state.
640         reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
641
642         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
643
644         expect_pending_htlcs_forwardable!(nodes[1]);
645         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
646         check_added_monitors!(nodes[1], 1);
647
648         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
649         assert_eq!(events.len(), 1);
650         let payment_event = SendEvent::from_event(events.pop().unwrap());
651         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
652         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
653         expect_pending_htlcs_forwardable!(nodes[2]);
654         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
655
656         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
657         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
658 }
659
660 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
661         // Test what happens if a node receives an MPP payment, claims it, but crashes before
662         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
663         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
664         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
665         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
666         // not have the preimage tied to the still-pending HTLC.
667         //
668         // To get to the correct state, on startup we should propagate the preimage to the
669         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
670         // receiving the preimage without a state update.
671         //
672         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
673         // definitely claimed.
674         let chanmon_cfgs = create_chanmon_cfgs(4);
675         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
676         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
677
678         let persister: test_utils::TestPersister;
679         let new_chain_monitor: test_utils::TestChainMonitor;
680         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
681
682         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
683
684         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
685         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
686         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
687         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
688
689         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
690         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
691         assert_eq!(route.paths.len(), 2);
692         route.paths.sort_by(|path_a, _| {
693                 // Sort the path so that the path through nodes[1] comes first
694                 if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
695                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
696         });
697
698         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
699         check_added_monitors!(nodes[0], 2);
700
701         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
702         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
703         assert_eq!(send_events.len(), 2);
704         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
705         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
706         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
707         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
708
709         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
710         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
711         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
712         if !persist_both_monitors {
713                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
714                         if outpoint.to_channel_id() == chan_id_not_persisted {
715                                 assert!(original_monitor.0.is_empty());
716                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
717                         }
718                 }
719         }
720
721         let original_manager = nodes[3].node.encode();
722
723         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
724
725         nodes[3].node.claim_funds(payment_preimage);
726         check_added_monitors!(nodes[3], 2);
727         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
728
729         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
730         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
731         // with the old ChannelManager.
732         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
733         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
734                 if outpoint.to_channel_id() == chan_id_persisted {
735                         assert!(updated_monitor.0.is_empty());
736                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
737                 }
738         }
739         // If `persist_both_monitors` is set, get the second monitor here as well
740         if persist_both_monitors {
741                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
742                         if outpoint.to_channel_id() == chan_id_not_persisted {
743                                 assert!(original_monitor.0.is_empty());
744                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
745                         }
746                 }
747         }
748
749         // Now restart nodes[3].
750         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
751
752         // On startup the preimage should have been copied into the non-persisted monitor:
753         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
754         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
755
756         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
757         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
758
759         // During deserialization, we should have closed one channel and broadcast its latest
760         // commitment transaction. We should also still have the original PaymentClaimable event we
761         // never finished processing.
762         let events = nodes[3].node.get_and_clear_pending_events();
763         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
764         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
765         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
766         if persist_both_monitors {
767                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
768         }
769
770         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
771         // ChannelManager prior to handling the original one.
772         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
773                 events[if persist_both_monitors { 3 } else { 2 }]
774         {
775                 assert_eq!(payment_hash, our_payment_hash);
776         } else { panic!(); }
777
778         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
779         if !persist_both_monitors {
780                 // If one of the two channels is still live, reveal the payment preimage over it.
781
782                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init { features: nodes[2].node.init_features(), remote_network_address: None }).unwrap();
783                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
784                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: nodes[3].node.init_features(), remote_network_address: None }).unwrap();
785                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
786
787                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
788                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
789                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
790
791                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
792
793                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
794                 // claim should fly.
795                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
796                 check_added_monitors!(nodes[3], 1);
797                 assert_eq!(ds_msgs.len(), 2);
798                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
799
800                 let cs_updates = match ds_msgs[1] {
801                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
802                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
803                                 check_added_monitors!(nodes[2], 1);
804                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
805                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
806                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
807                                 cs_updates
808                         }
809                         _ => panic!(),
810                 };
811
812                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
813                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
814                 expect_payment_sent!(nodes[0], payment_preimage);
815         }
816 }
817
818 #[test]
819 fn test_partial_claim_before_restart() {
820         do_test_partial_claim_before_restart(false);
821         do_test_partial_claim_before_restart(true);
822 }
823
824 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
825         if !use_cs_commitment { assert!(!claim_htlc); }
826         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
827         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
828         // it back until the ChannelMonitor decides the fate of the HTLC.
829         // This was never an issue, but it may be easy to regress here going forward.
830         let chanmon_cfgs = create_chanmon_cfgs(3);
831         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
832         let mut intercept_forwards_config = test_default_channel_config();
833         intercept_forwards_config.accept_intercept_htlcs = true;
834         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
835
836         let persister;
837         let new_chain_monitor;
838         let nodes_1_deserialized;
839
840         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
841
842         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
843         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
844
845         let intercept_scid = nodes[1].node.get_intercept_scid();
846
847         let (mut route, payment_hash, payment_preimage, payment_secret) =
848                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
849         if use_intercept {
850                 route.paths[0][1].short_channel_id = intercept_scid;
851         }
852         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
853         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
854         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
855         check_added_monitors!(nodes[0], 1);
856
857         let payment_event = SendEvent::from_node(&nodes[0]);
858         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
859         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
860
861         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
862         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
863         // present when we serialized.
864         let node_encoded = nodes[1].node.encode();
865
866         let mut intercept_id = None;
867         let mut expected_outbound_amount_msat = None;
868         if use_intercept {
869                 let events = nodes[1].node.get_and_clear_pending_events();
870                 assert_eq!(events.len(), 1);
871                 match events[0] {
872                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
873                                 intercept_id = Some(ev_id);
874                                 expected_outbound_amount_msat = Some(ev_amt);
875                         },
876                         _ => panic!()
877                 }
878                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
879                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
880         }
881
882         expect_pending_htlcs_forwardable!(nodes[1]);
883
884         let payment_event = SendEvent::from_node(&nodes[1]);
885         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
886         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
887         check_added_monitors!(nodes[2], 1);
888
889         if claim_htlc {
890                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
891                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
892         }
893         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
894
895         let _ = nodes[2].node.get_and_clear_pending_msg_events();
896
897         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
898         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
899         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
900
901         check_added_monitors!(nodes[2], 1);
902         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
903         check_closed_broadcast!(nodes[2], true);
904
905         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
906         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
907         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
908
909         // Note that this checks that this is the only event on nodes[1], implying the
910         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
911         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
912
913         if use_intercept {
914                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
915                 // a intercept-doesn't-exist error.
916                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
917                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
918                 assert_eq!(forward_err, APIError::APIMisuseError {
919                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
920                 });
921         }
922
923         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
924         assert_eq!(bs_commitment_tx.len(), 1);
925
926         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
927         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
928
929         if use_cs_commitment {
930                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
931                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
932                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
933                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
934                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
935
936                 if claim_htlc {
937                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
938                 } else {
939                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
940                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
941                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
942                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
943                 }
944         } else {
945                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
946         }
947
948         if !claim_htlc {
949                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
950         } else {
951                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
952         }
953         check_added_monitors!(nodes[1], 1);
954
955         let events = nodes[1].node.get_and_clear_pending_msg_events();
956         assert_eq!(events.len(), 1);
957         match &events[0] {
958                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
959                         if claim_htlc {
960                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
961                         } else {
962                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
963                         }
964                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
965                 },
966                 _ => panic!("Unexpected event"),
967         }
968
969         if claim_htlc {
970                 expect_payment_sent!(nodes[0], payment_preimage);
971         } else {
972                 expect_payment_failed!(nodes[0], payment_hash, false);
973         }
974 }
975
976 #[test]
977 fn forwarded_payment_no_manager_persistence() {
978         do_forwarded_payment_no_manager_persistence(true, true, false);
979         do_forwarded_payment_no_manager_persistence(true, false, false);
980         do_forwarded_payment_no_manager_persistence(false, false, false);
981 }
982
983 #[test]
984 fn intercepted_payment_no_manager_persistence() {
985         do_forwarded_payment_no_manager_persistence(true, true, true);
986         do_forwarded_payment_no_manager_persistence(true, false, true);
987         do_forwarded_payment_no_manager_persistence(false, false, true);
988 }
989
990 #[test]
991 fn removed_payment_no_manager_persistence() {
992         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
993         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
994         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
995         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
996         // were left dangling when a channel was force-closed due to a stale ChannelManager.
997         let chanmon_cfgs = create_chanmon_cfgs(3);
998         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
999         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1000
1001         let persister;
1002         let new_chain_monitor;
1003         let nodes_1_deserialized;
1004
1005         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1006
1007         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1008         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1009
1010         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1011
1012         let node_encoded = nodes[1].node.encode();
1013
1014         nodes[2].node.fail_htlc_backwards(&payment_hash);
1015         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1016         check_added_monitors!(nodes[2], 1);
1017         let events = nodes[2].node.get_and_clear_pending_msg_events();
1018         assert_eq!(events.len(), 1);
1019         match &events[0] {
1020                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1021                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1022                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1023                 },
1024                 _ => panic!("Unexpected event"),
1025         }
1026
1027         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1028         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1029         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1030
1031         match nodes[1].node.pop_pending_event().unwrap() {
1032                 Event::ChannelClosed { ref reason, .. } => {
1033                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1034                 },
1035                 _ => panic!("Unexpected event"),
1036         }
1037
1038         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1039         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1040         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1041         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1042         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1043
1044         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1045         check_added_monitors!(nodes[1], 1);
1046         let events = nodes[1].node.get_and_clear_pending_msg_events();
1047         assert_eq!(events.len(), 1);
1048         match &events[0] {
1049                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1050                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1051                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1052                 },
1053                 _ => panic!("Unexpected event"),
1054         }
1055
1056         expect_payment_failed!(nodes[0], payment_hash, false);
1057 }