]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
DRY up repeated HTLC `Balance`s in tests
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::ChannelMonitor;
15 use crate::chain::keysinterface::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
19 use crate::ln::msgs;
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::util::enforcing_trait_impls::EnforcingSigner;
22 use crate::util::test_utils;
23 use crate::util::errors::APIError;
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26 use crate::util::string::UntrustedString;
27
28 use bitcoin::hash_types::BlockHash;
29
30 use crate::prelude::*;
31 use core::default::Default;
32 use crate::sync::Mutex;
33
34 use crate::ln::functional_test_utils::*;
35
36 #[test]
37 fn test_funding_peer_disconnect() {
38         // Test that we can lock in our funding tx while disconnected
39         let chanmon_cfgs = create_chanmon_cfgs(2);
40         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
41         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
42         let persister: test_utils::TestPersister;
43         let new_chain_monitor: test_utils::TestChainMonitor;
44         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
45         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
46         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
47
48         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
49         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
50
51         confirm_transaction(&nodes[0], &tx);
52         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
53         assert!(events_1.is_empty());
54
55         reconnect_nodes(&nodes[0], &nodes[1], (false, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
56
57         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
58         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
59
60         confirm_transaction(&nodes[1], &tx);
61         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
62         assert!(events_2.is_empty());
63
64         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }, true).unwrap();
65         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
66         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, false).unwrap();
67         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
68
69         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
70         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
71         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
72         assert_eq!(events_3.len(), 1);
73         let as_channel_ready = match events_3[0] {
74                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
75                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
76                         msg.clone()
77                 },
78                 _ => panic!("Unexpected event {:?}", events_3[0]),
79         };
80
81         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
82         // announcement_signatures as well as channel_update.
83         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
84         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
85         assert_eq!(events_4.len(), 3);
86         let chan_id;
87         let bs_channel_ready = match events_4[0] {
88                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
89                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
90                         chan_id = msg.channel_id;
91                         msg.clone()
92                 },
93                 _ => panic!("Unexpected event {:?}", events_4[0]),
94         };
95         let bs_announcement_sigs = match events_4[1] {
96                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
97                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
98                         msg.clone()
99                 },
100                 _ => panic!("Unexpected event {:?}", events_4[1]),
101         };
102         match events_4[2] {
103                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
104                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
105                 },
106                 _ => panic!("Unexpected event {:?}", events_4[2]),
107         }
108
109         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
110         // generates a duplicative private channel_update
111         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
112         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
113         assert_eq!(events_5.len(), 1);
114         match events_5[0] {
115                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
116                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
117                 },
118                 _ => panic!("Unexpected event {:?}", events_5[0]),
119         };
120
121         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
122         // announcement_signatures.
123         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
124         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
125         assert_eq!(events_6.len(), 1);
126         let as_announcement_sigs = match events_6[0] {
127                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
128                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
129                         msg.clone()
130                 },
131                 _ => panic!("Unexpected event {:?}", events_6[0]),
132         };
133         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
134         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
135
136         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
137         // broadcast the channel announcement globally, as well as re-send its (now-public)
138         // channel_update.
139         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
140         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
141         assert_eq!(events_7.len(), 1);
142         let (chan_announcement, as_update) = match events_7[0] {
143                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
144                         (msg.clone(), update_msg.clone().unwrap())
145                 },
146                 _ => panic!("Unexpected event {:?}", events_7[0]),
147         };
148
149         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
150         // same channel_announcement.
151         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
152         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
153         assert_eq!(events_8.len(), 1);
154         let bs_update = match events_8[0] {
155                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
156                         assert_eq!(*msg, chan_announcement);
157                         update_msg.clone().unwrap()
158                 },
159                 _ => panic!("Unexpected event {:?}", events_8[0]),
160         };
161
162         // Provide the channel announcement and public updates to the network graph
163         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
164         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
165         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
166
167         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
168         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
169         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
170
171         // Check that after deserialization and reconnection we can still generate an identical
172         // channel_announcement from the cached signatures.
173         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
174
175         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
176
177         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
178
179         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
180 }
181
182 #[test]
183 fn test_no_txn_manager_serialize_deserialize() {
184         let chanmon_cfgs = create_chanmon_cfgs(2);
185         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
186         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
187         let persister: test_utils::TestPersister;
188         let new_chain_monitor: test_utils::TestChainMonitor;
189         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
190         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
191
192         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
193
194         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
195
196         let chan_0_monitor_serialized =
197                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
198         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
199
200         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }, true).unwrap();
201         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
202         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, false).unwrap();
203         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
204
205         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
206         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
207         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
208         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
209
210         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
211         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
212         for node in nodes.iter() {
213                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
214                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
215                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
216         }
217
218         send_payment(&nodes[0], &[&nodes[1]], 1000000);
219 }
220
221 #[test]
222 fn test_manager_serialize_deserialize_events() {
223         // This test makes sure the events field in ChannelManager survives de/serialization
224         let chanmon_cfgs = create_chanmon_cfgs(2);
225         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
226         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
227         let persister: test_utils::TestPersister;
228         let new_chain_monitor: test_utils::TestChainMonitor;
229         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
230         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
231
232         // Start creating a channel, but stop right before broadcasting the funding transaction
233         let channel_value = 100000;
234         let push_msat = 10001;
235         let node_a = nodes.remove(0);
236         let node_b = nodes.remove(0);
237         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
238         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
239         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
240
241         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
242
243         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
244         check_added_monitors!(node_a, 0);
245
246         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
247         {
248                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
249                 assert_eq!(added_monitors.len(), 1);
250                 assert_eq!(added_monitors[0].0, funding_output);
251                 added_monitors.clear();
252         }
253
254         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
255         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
256         {
257                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
258                 assert_eq!(added_monitors.len(), 1);
259                 assert_eq!(added_monitors[0].0, funding_output);
260                 added_monitors.clear();
261         }
262         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
263
264         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
265         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
266
267         nodes.push(node_a);
268         nodes.push(node_b);
269
270         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
271         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
272         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
273
274         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
275
276         // After deserializing, make sure the funding_transaction is still held by the channel manager
277         let events_4 = nodes[0].node.get_and_clear_pending_events();
278         assert_eq!(events_4.len(), 0);
279         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
280         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
281
282         // Make sure the channel is functioning as though the de/serialization never happened
283         assert_eq!(nodes[0].node.list_channels().len(), 1);
284
285         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }, true).unwrap();
286         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
287         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, false).unwrap();
288         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
289
290         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
291         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
292         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
293         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
294
295         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
296         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
297         for node in nodes.iter() {
298                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
299                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
300                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
301         }
302
303         send_payment(&nodes[0], &[&nodes[1]], 1000000);
304 }
305
306 #[test]
307 fn test_simple_manager_serialize_deserialize() {
308         let chanmon_cfgs = create_chanmon_cfgs(2);
309         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
310         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
311         let persister: test_utils::TestPersister;
312         let new_chain_monitor: test_utils::TestChainMonitor;
313         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
314         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
315         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
316
317         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
318         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
319
320         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
321
322         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
323         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
324
325         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
326
327         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
328         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
329 }
330
331 #[test]
332 fn test_manager_serialize_deserialize_inconsistent_monitor() {
333         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
334         let chanmon_cfgs = create_chanmon_cfgs(4);
335         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
336         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
337         let logger: test_utils::TestLogger;
338         let fee_estimator: test_utils::TestFeeEstimator;
339         let persister: test_utils::TestPersister;
340         let new_chain_monitor: test_utils::TestChainMonitor;
341         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
342         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
343         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
344         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
345         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
346
347         let mut node_0_stale_monitors_serialized = Vec::new();
348         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
349                 let mut writer = test_utils::TestVecWriter(Vec::new());
350                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
351                 node_0_stale_monitors_serialized.push(writer.0);
352         }
353
354         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
355
356         // Serialize the ChannelManager here, but the monitor we keep up-to-date
357         let nodes_0_serialized = nodes[0].node.encode();
358
359         route_payment(&nodes[0], &[&nodes[3]], 1000000);
360         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
361         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
362         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
363
364         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
365         // nodes[3])
366         let mut node_0_monitors_serialized = Vec::new();
367         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
368                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
369         }
370
371         logger = test_utils::TestLogger::new();
372         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
373         persister = test_utils::TestPersister::new();
374         let keys_manager = &chanmon_cfgs[0].keys_manager;
375         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster.clone(), &logger, &fee_estimator, &persister, keys_manager);
376         nodes[0].chain_monitor = &new_chain_monitor;
377
378
379         let mut node_0_stale_monitors = Vec::new();
380         for serialized in node_0_stale_monitors_serialized.iter() {
381                 let mut read = &serialized[..];
382                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
383                 assert!(read.is_empty());
384                 node_0_stale_monitors.push(monitor);
385         }
386
387         let mut node_0_monitors = Vec::new();
388         for serialized in node_0_monitors_serialized.iter() {
389                 let mut read = &serialized[..];
390                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
391                 assert!(read.is_empty());
392                 node_0_monitors.push(monitor);
393         }
394
395         let mut nodes_0_read = &nodes_0_serialized[..];
396         if let Err(msgs::DecodeError::InvalidValue) =
397                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
398                 default_config: UserConfig::default(),
399                 entropy_source: keys_manager,
400                 node_signer: keys_manager,
401                 signer_provider: keys_manager,
402                 fee_estimator: &fee_estimator,
403                 router: &nodes[0].router,
404                 chain_monitor: nodes[0].chain_monitor,
405                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
406                 logger: &logger,
407                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
408         }) { } else {
409                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
410         };
411
412         let mut nodes_0_read = &nodes_0_serialized[..];
413         let (_, nodes_0_deserialized_tmp) =
414                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
415                 default_config: UserConfig::default(),
416                 entropy_source: keys_manager,
417                 node_signer: keys_manager,
418                 signer_provider: keys_manager,
419                 fee_estimator: &fee_estimator,
420                 router: nodes[0].router,
421                 chain_monitor: nodes[0].chain_monitor,
422                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
423                 logger: &logger,
424                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
425         }).unwrap();
426         nodes_0_deserialized = nodes_0_deserialized_tmp;
427         assert!(nodes_0_read.is_empty());
428
429         for monitor in node_0_monitors.drain(..) {
430                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
431                         ChannelMonitorUpdateStatus::Completed);
432                 check_added_monitors!(nodes[0], 1);
433         }
434         nodes[0].node = &nodes_0_deserialized;
435
436         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
437         { // Channel close should result in a commitment tx
438                 nodes[0].node.timer_tick_occurred();
439                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
440                 assert_eq!(txn.len(), 1);
441                 check_spends!(txn[0], funding_tx);
442                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
443         }
444         check_added_monitors!(nodes[0], 1);
445
446         // nodes[1] and nodes[2] have no lost state with nodes[0]...
447         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
448         reconnect_nodes(&nodes[0], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
449         //... and we can even still claim the payment!
450         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
451
452         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, true).unwrap();
453         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
454         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: nodes[3].node.init_features(), remote_network_address: None }, false).unwrap();
455         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
456         let mut found_err = false;
457         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
458                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
459                         match action {
460                                 &ErrorAction::SendErrorMessage { ref msg } => {
461                                         assert_eq!(msg.channel_id, channel_id);
462                                         assert!(!found_err);
463                                         found_err = true;
464                                 },
465                                 _ => panic!("Unexpected event!"),
466                         }
467                 }
468         }
469         assert!(found_err);
470 }
471
472 fn do_test_data_loss_protect(reconnect_panicing: bool) {
473         // When we get a data_loss_protect proving we're behind, we immediately panic as the
474         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
475         // panic message informs the user they should force-close without broadcasting, which is tested
476         // if `reconnect_panicing` is not set.
477         let mut chanmon_cfgs = create_chanmon_cfgs(2);
478         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
479         // during signing due to revoked tx
480         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
481         let persister;
482         let new_chain_monitor;
483         let nodes_0_deserialized;
484         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
485         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
486         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
487
488         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
489
490         // Cache node A state before any channel update
491         let previous_node_state = nodes[0].node.encode();
492         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
493
494         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
495         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
496
497         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
498         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
499
500         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
501
502         if reconnect_panicing {
503                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }, true).unwrap();
504                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, false).unwrap();
505
506                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
507
508                 // Check we close channel detecting A is fallen-behind
509                 // Check that we sent the warning message when we detected that A has fallen behind,
510                 // and give the possibility for A to recover from the warning.
511                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
512                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
513                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
514
515                 {
516                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
517                         // The node B should not broadcast the transaction to force close the channel!
518                         assert!(node_txn.is_empty());
519                 }
520
521                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
522                 // Check A panics upon seeing proof it has fallen behind.
523                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
524                 return; // By this point we should have panic'ed!
525         }
526
527         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
528         check_added_monitors!(nodes[0], 1);
529         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
530         {
531                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
532                 assert_eq!(node_txn.len(), 0);
533         }
534
535         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
536                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
537                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
538                         match action {
539                                 &ErrorAction::SendErrorMessage { ref msg } => {
540                                         assert_eq!(msg.data, "Channel force-closed");
541                                 },
542                                 _ => panic!("Unexpected event!"),
543                         }
544                 } else {
545                         panic!("Unexpected event {:?}", msg)
546                 }
547         }
548
549         // after the warning message sent by B, we should not able to
550         // use the channel, or reconnect with success to the channel.
551         assert!(nodes[0].node.list_usable_channels().is_empty());
552         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: nodes[1].node.init_features(), remote_network_address: None }, true).unwrap();
553         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: nodes[0].node.init_features(), remote_network_address: None }, false).unwrap();
554         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
555
556         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
557         let mut err_msgs_0 = Vec::with_capacity(1);
558         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
559                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
560                         match action {
561                                 &ErrorAction::SendErrorMessage { ref msg } => {
562                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
563                                         err_msgs_0.push(msg.clone());
564                                 },
565                                 _ => panic!("Unexpected event!"),
566                         }
567                 } else {
568                         panic!("Unexpected event!");
569                 }
570         }
571         assert_eq!(err_msgs_0.len(), 1);
572         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
573         assert!(nodes[1].node.list_usable_channels().is_empty());
574         check_added_monitors!(nodes[1], 1);
575         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
576         check_closed_broadcast!(nodes[1], false);
577 }
578
579 #[test]
580 #[should_panic]
581 fn test_data_loss_protect_showing_stale_state_panics() {
582         do_test_data_loss_protect(true);
583 }
584
585 #[test]
586 fn test_force_close_without_broadcast() {
587         do_test_data_loss_protect(false);
588 }
589
590 #[test]
591 fn test_forwardable_regen() {
592         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
593         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
594         // HTLCs.
595         // We test it for both payment receipt and payment forwarding.
596
597         let chanmon_cfgs = create_chanmon_cfgs(3);
598         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
599         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
600         let persister: test_utils::TestPersister;
601         let new_chain_monitor: test_utils::TestChainMonitor;
602         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
603         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
604         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
605         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
606
607         // First send a payment to nodes[1]
608         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
609         nodes[0].node.send_payment_with_route(&route, payment_hash,
610                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
611         check_added_monitors!(nodes[0], 1);
612
613         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
614         assert_eq!(events.len(), 1);
615         let payment_event = SendEvent::from_event(events.pop().unwrap());
616         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
617         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
618
619         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
620
621         // Next send a payment which is forwarded by nodes[1]
622         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
623         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
624                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
625         check_added_monitors!(nodes[0], 1);
626
627         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
628         assert_eq!(events.len(), 1);
629         let payment_event = SendEvent::from_event(events.pop().unwrap());
630         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
631         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
632
633         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
634         // generated
635         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
636
637         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
638         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
639         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
640
641         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
642         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
643         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
644
645         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
646         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
647         // the commitment state.
648         reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
649
650         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
651
652         expect_pending_htlcs_forwardable!(nodes[1]);
653         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
654         check_added_monitors!(nodes[1], 1);
655
656         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
657         assert_eq!(events.len(), 1);
658         let payment_event = SendEvent::from_event(events.pop().unwrap());
659         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
660         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
661         expect_pending_htlcs_forwardable!(nodes[2]);
662         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
663
664         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
665         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
666 }
667
668 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
669         // Test what happens if a node receives an MPP payment, claims it, but crashes before
670         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
671         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
672         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
673         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
674         // not have the preimage tied to the still-pending HTLC.
675         //
676         // To get to the correct state, on startup we should propagate the preimage to the
677         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
678         // receiving the preimage without a state update.
679         //
680         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
681         // definitely claimed.
682         let chanmon_cfgs = create_chanmon_cfgs(4);
683         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
684         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
685
686         let persister: test_utils::TestPersister;
687         let new_chain_monitor: test_utils::TestChainMonitor;
688         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
689
690         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
691
692         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
693         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
694         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
695         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
696
697         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
698         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
699         assert_eq!(route.paths.len(), 2);
700         route.paths.sort_by(|path_a, _| {
701                 // Sort the path so that the path through nodes[1] comes first
702                 if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
703                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
704         });
705
706         nodes[0].node.send_payment_with_route(&route, payment_hash,
707                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
708         check_added_monitors!(nodes[0], 2);
709
710         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
711         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
712         assert_eq!(send_events.len(), 2);
713         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
714         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
715         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
716         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
717
718         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
719         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
720         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
721         if !persist_both_monitors {
722                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
723                         if outpoint.to_channel_id() == chan_id_not_persisted {
724                                 assert!(original_monitor.0.is_empty());
725                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
726                         }
727                 }
728         }
729
730         let original_manager = nodes[3].node.encode();
731
732         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
733
734         nodes[3].node.claim_funds(payment_preimage);
735         check_added_monitors!(nodes[3], 2);
736         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
737
738         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
739         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
740         // with the old ChannelManager.
741         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
742         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
743                 if outpoint.to_channel_id() == chan_id_persisted {
744                         assert!(updated_monitor.0.is_empty());
745                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
746                 }
747         }
748         // If `persist_both_monitors` is set, get the second monitor here as well
749         if persist_both_monitors {
750                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
751                         if outpoint.to_channel_id() == chan_id_not_persisted {
752                                 assert!(original_monitor.0.is_empty());
753                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
754                         }
755                 }
756         }
757
758         // Now restart nodes[3].
759         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
760
761         // On startup the preimage should have been copied into the non-persisted monitor:
762         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
763         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
764
765         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
766         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
767
768         // During deserialization, we should have closed one channel and broadcast its latest
769         // commitment transaction. We should also still have the original PaymentClaimable event we
770         // never finished processing.
771         let events = nodes[3].node.get_and_clear_pending_events();
772         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
773         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
774         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
775         if persist_both_monitors {
776                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
777         }
778
779         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
780         // ChannelManager prior to handling the original one.
781         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
782                 events[if persist_both_monitors { 3 } else { 2 }]
783         {
784                 assert_eq!(payment_hash, our_payment_hash);
785         } else { panic!(); }
786
787         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
788         if !persist_both_monitors {
789                 // If one of the two channels is still live, reveal the payment preimage over it.
790
791                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init { features: nodes[2].node.init_features(), remote_network_address: None }, true).unwrap();
792                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
793                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: nodes[3].node.init_features(), remote_network_address: None }, false).unwrap();
794                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
795
796                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
797                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
798                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
799
800                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
801
802                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
803                 // claim should fly.
804                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
805                 check_added_monitors!(nodes[3], 1);
806                 assert_eq!(ds_msgs.len(), 2);
807                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
808
809                 let cs_updates = match ds_msgs[1] {
810                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
811                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
812                                 check_added_monitors!(nodes[2], 1);
813                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
814                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
815                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
816                                 cs_updates
817                         }
818                         _ => panic!(),
819                 };
820
821                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
822                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
823                 expect_payment_sent!(nodes[0], payment_preimage);
824         }
825 }
826
827 #[test]
828 fn test_partial_claim_before_restart() {
829         do_test_partial_claim_before_restart(false);
830         do_test_partial_claim_before_restart(true);
831 }
832
833 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
834         if !use_cs_commitment { assert!(!claim_htlc); }
835         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
836         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
837         // it back until the ChannelMonitor decides the fate of the HTLC.
838         // This was never an issue, but it may be easy to regress here going forward.
839         let chanmon_cfgs = create_chanmon_cfgs(3);
840         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
841         let mut intercept_forwards_config = test_default_channel_config();
842         intercept_forwards_config.accept_intercept_htlcs = true;
843         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
844
845         let persister;
846         let new_chain_monitor;
847         let nodes_1_deserialized;
848
849         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
850
851         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
852         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
853
854         let intercept_scid = nodes[1].node.get_intercept_scid();
855
856         let (mut route, payment_hash, payment_preimage, payment_secret) =
857                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
858         if use_intercept {
859                 route.paths[0][1].short_channel_id = intercept_scid;
860         }
861         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
862         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
863         nodes[0].node.send_payment_with_route(&route, payment_hash,
864                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
865         check_added_monitors!(nodes[0], 1);
866
867         let payment_event = SendEvent::from_node(&nodes[0]);
868         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
869         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
870
871         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
872         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
873         // present when we serialized.
874         let node_encoded = nodes[1].node.encode();
875
876         let mut intercept_id = None;
877         let mut expected_outbound_amount_msat = None;
878         if use_intercept {
879                 let events = nodes[1].node.get_and_clear_pending_events();
880                 assert_eq!(events.len(), 1);
881                 match events[0] {
882                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
883                                 intercept_id = Some(ev_id);
884                                 expected_outbound_amount_msat = Some(ev_amt);
885                         },
886                         _ => panic!()
887                 }
888                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
889                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
890         }
891
892         expect_pending_htlcs_forwardable!(nodes[1]);
893
894         let payment_event = SendEvent::from_node(&nodes[1]);
895         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
896         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
897         check_added_monitors!(nodes[2], 1);
898
899         if claim_htlc {
900                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
901                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
902         }
903         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
904
905         let _ = nodes[2].node.get_and_clear_pending_msg_events();
906
907         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
908         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
909         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
910
911         check_added_monitors!(nodes[2], 1);
912         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
913         check_closed_broadcast!(nodes[2], true);
914
915         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
916         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
917         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
918
919         // Note that this checks that this is the only event on nodes[1], implying the
920         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
921         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
922
923         if use_intercept {
924                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
925                 // a intercept-doesn't-exist error.
926                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
927                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
928                 assert_eq!(forward_err, APIError::APIMisuseError {
929                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
930                 });
931         }
932
933         nodes[1].node.timer_tick_occurred();
934         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
935         assert_eq!(bs_commitment_tx.len(), 1);
936         check_added_monitors!(nodes[1], 1);
937
938         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
939         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
940
941         if use_cs_commitment {
942                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
943                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
944                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
945                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
946                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
947
948                 if claim_htlc {
949                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
950                 } else {
951                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
952                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
953                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
954                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
955                 }
956         } else {
957                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
958         }
959
960         if !claim_htlc {
961                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
962         } else {
963                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
964         }
965         check_added_monitors!(nodes[1], 1);
966
967         let events = nodes[1].node.get_and_clear_pending_msg_events();
968         assert_eq!(events.len(), 1);
969         match &events[0] {
970                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
971                         if claim_htlc {
972                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
973                         } else {
974                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
975                         }
976                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
977                 },
978                 _ => panic!("Unexpected event"),
979         }
980
981         if claim_htlc {
982                 expect_payment_sent!(nodes[0], payment_preimage);
983         } else {
984                 expect_payment_failed!(nodes[0], payment_hash, false);
985         }
986 }
987
988 #[test]
989 fn forwarded_payment_no_manager_persistence() {
990         do_forwarded_payment_no_manager_persistence(true, true, false);
991         do_forwarded_payment_no_manager_persistence(true, false, false);
992         do_forwarded_payment_no_manager_persistence(false, false, false);
993 }
994
995 #[test]
996 fn intercepted_payment_no_manager_persistence() {
997         do_forwarded_payment_no_manager_persistence(true, true, true);
998         do_forwarded_payment_no_manager_persistence(true, false, true);
999         do_forwarded_payment_no_manager_persistence(false, false, true);
1000 }
1001
1002 #[test]
1003 fn removed_payment_no_manager_persistence() {
1004         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1005         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1006         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1007         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1008         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1009         let chanmon_cfgs = create_chanmon_cfgs(3);
1010         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1011         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1012
1013         let persister;
1014         let new_chain_monitor;
1015         let nodes_1_deserialized;
1016
1017         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1018
1019         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1020         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1021
1022         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1023
1024         let node_encoded = nodes[1].node.encode();
1025
1026         nodes[2].node.fail_htlc_backwards(&payment_hash);
1027         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1028         check_added_monitors!(nodes[2], 1);
1029         let events = nodes[2].node.get_and_clear_pending_msg_events();
1030         assert_eq!(events.len(), 1);
1031         match &events[0] {
1032                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1033                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1034                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1035                 },
1036                 _ => panic!("Unexpected event"),
1037         }
1038
1039         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1040         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1041         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1042
1043         match nodes[1].node.pop_pending_event().unwrap() {
1044                 Event::ChannelClosed { ref reason, .. } => {
1045                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1046                 },
1047                 _ => panic!("Unexpected event"),
1048         }
1049
1050         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1051         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1052         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1053         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1054         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1055
1056         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1057         check_added_monitors!(nodes[1], 1);
1058         let events = nodes[1].node.get_and_clear_pending_msg_events();
1059         assert_eq!(events.len(), 1);
1060         match &events[0] {
1061                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1062                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1063                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1064                 },
1065                 _ => panic!("Unexpected event"),
1066         }
1067
1068         expect_payment_failed!(nodes[0], payment_hash, false);
1069 }