]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
Add counterparty_node_id & channel_capacity to ChannelClosed
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::ChannelMonitor;
15 use crate::sign::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
19 use crate::ln::msgs;
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::util::enforcing_trait_impls::EnforcingSigner;
22 use crate::util::test_utils;
23 use crate::util::errors::APIError;
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26 use crate::util::string::UntrustedString;
27
28 use bitcoin::hash_types::BlockHash;
29
30 use crate::prelude::*;
31 use core::default::Default;
32 use crate::sync::Mutex;
33
34 use crate::ln::functional_test_utils::*;
35
36 #[test]
37 fn test_funding_peer_disconnect() {
38         // Test that we can lock in our funding tx while disconnected
39         let chanmon_cfgs = create_chanmon_cfgs(2);
40         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
41         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
42         let persister: test_utils::TestPersister;
43         let new_chain_monitor: test_utils::TestChainMonitor;
44         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
45         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
46         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
47
48         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
49         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
50
51         confirm_transaction(&nodes[0], &tx);
52         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
53         assert!(events_1.is_empty());
54
55         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
56         reconnect_args.send_channel_ready.1 = true;
57         reconnect_nodes(reconnect_args);
58
59         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
60         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
61
62         confirm_transaction(&nodes[1], &tx);
63         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
64         assert!(events_2.is_empty());
65
66         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
67                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
68         }, true).unwrap();
69         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
70         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
71                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
72         }, false).unwrap();
73         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
74
75         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
76         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
77         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
78         assert_eq!(events_3.len(), 1);
79         let as_channel_ready = match events_3[0] {
80                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
81                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
82                         msg.clone()
83                 },
84                 _ => panic!("Unexpected event {:?}", events_3[0]),
85         };
86
87         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
88         // announcement_signatures as well as channel_update.
89         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
90         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
91         assert_eq!(events_4.len(), 3);
92         let chan_id;
93         let bs_channel_ready = match events_4[0] {
94                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
95                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
96                         chan_id = msg.channel_id;
97                         msg.clone()
98                 },
99                 _ => panic!("Unexpected event {:?}", events_4[0]),
100         };
101         let bs_announcement_sigs = match events_4[1] {
102                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
103                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
104                         msg.clone()
105                 },
106                 _ => panic!("Unexpected event {:?}", events_4[1]),
107         };
108         match events_4[2] {
109                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
110                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
111                 },
112                 _ => panic!("Unexpected event {:?}", events_4[2]),
113         }
114
115         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
116         // generates a duplicative private channel_update
117         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
118         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
119         assert_eq!(events_5.len(), 1);
120         match events_5[0] {
121                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
122                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
123                 },
124                 _ => panic!("Unexpected event {:?}", events_5[0]),
125         };
126
127         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
128         // announcement_signatures.
129         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
130         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
131         assert_eq!(events_6.len(), 1);
132         let as_announcement_sigs = match events_6[0] {
133                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
134                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
135                         msg.clone()
136                 },
137                 _ => panic!("Unexpected event {:?}", events_6[0]),
138         };
139         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
140         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
141
142         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
143         // broadcast the channel announcement globally, as well as re-send its (now-public)
144         // channel_update.
145         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
146         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
147         assert_eq!(events_7.len(), 1);
148         let (chan_announcement, as_update) = match events_7[0] {
149                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
150                         (msg.clone(), update_msg.clone().unwrap())
151                 },
152                 _ => panic!("Unexpected event {:?}", events_7[0]),
153         };
154
155         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
156         // same channel_announcement.
157         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
158         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
159         assert_eq!(events_8.len(), 1);
160         let bs_update = match events_8[0] {
161                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
162                         assert_eq!(*msg, chan_announcement);
163                         update_msg.clone().unwrap()
164                 },
165                 _ => panic!("Unexpected event {:?}", events_8[0]),
166         };
167
168         // Provide the channel announcement and public updates to the network graph
169         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
170         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
171         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
172
173         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
174         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
175         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
176
177         // Check that after deserialization and reconnection we can still generate an identical
178         // channel_announcement from the cached signatures.
179         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
180
181         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
182
183         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
184
185         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
186 }
187
188 #[test]
189 fn test_no_txn_manager_serialize_deserialize() {
190         let chanmon_cfgs = create_chanmon_cfgs(2);
191         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
192         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
193         let persister: test_utils::TestPersister;
194         let new_chain_monitor: test_utils::TestChainMonitor;
195         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
196         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
197
198         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
199
200         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
201
202         let chan_0_monitor_serialized =
203                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
204         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
205
206         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
207                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
208         }, true).unwrap();
209         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
210         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
211                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
212         }, false).unwrap();
213         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
214
215         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
216         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
217         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
218         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
219
220         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
221         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
222         for node in nodes.iter() {
223                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
224                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
225                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
226         }
227
228         send_payment(&nodes[0], &[&nodes[1]], 1000000);
229 }
230
231 #[test]
232 fn test_manager_serialize_deserialize_events() {
233         // This test makes sure the events field in ChannelManager survives de/serialization
234         let chanmon_cfgs = create_chanmon_cfgs(2);
235         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
236         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
237         let persister: test_utils::TestPersister;
238         let new_chain_monitor: test_utils::TestChainMonitor;
239         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
240         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
241
242         // Start creating a channel, but stop right before broadcasting the funding transaction
243         let channel_value = 100000;
244         let push_msat = 10001;
245         let node_a = nodes.remove(0);
246         let node_b = nodes.remove(0);
247         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
248         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
249         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
250
251         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
252
253         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
254         check_added_monitors!(node_a, 0);
255
256         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
257         {
258                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
259                 assert_eq!(added_monitors.len(), 1);
260                 assert_eq!(added_monitors[0].0, funding_output);
261                 added_monitors.clear();
262         }
263
264         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
265         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
266         {
267                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
268                 assert_eq!(added_monitors.len(), 1);
269                 assert_eq!(added_monitors[0].0, funding_output);
270                 added_monitors.clear();
271         }
272         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
273
274         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
275         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
276
277         nodes.push(node_a);
278         nodes.push(node_b);
279
280         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
281         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
282         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
283
284         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
285
286         // After deserializing, make sure the funding_transaction is still held by the channel manager
287         let events_4 = nodes[0].node.get_and_clear_pending_events();
288         assert_eq!(events_4.len(), 0);
289         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
290         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
291
292         // Make sure the channel is functioning as though the de/serialization never happened
293         assert_eq!(nodes[0].node.list_channels().len(), 1);
294
295         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
296                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
297         }, true).unwrap();
298         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
299         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
300                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
301         }, false).unwrap();
302         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
303
304         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
305         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
306         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
307         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
308
309         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
310         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
311         for node in nodes.iter() {
312                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
313                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
314                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
315         }
316
317         send_payment(&nodes[0], &[&nodes[1]], 1000000);
318 }
319
320 #[test]
321 fn test_simple_manager_serialize_deserialize() {
322         let chanmon_cfgs = create_chanmon_cfgs(2);
323         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
324         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
325         let persister: test_utils::TestPersister;
326         let new_chain_monitor: test_utils::TestChainMonitor;
327         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
328         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
329         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
330
331         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
332         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
333
334         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
335
336         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
337         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
338
339         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
340
341         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
342         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
343 }
344
345 #[test]
346 fn test_manager_serialize_deserialize_inconsistent_monitor() {
347         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
348         let chanmon_cfgs = create_chanmon_cfgs(4);
349         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
350         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
351         let logger: test_utils::TestLogger;
352         let fee_estimator: test_utils::TestFeeEstimator;
353         let persister: test_utils::TestPersister;
354         let new_chain_monitor: test_utils::TestChainMonitor;
355         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
356         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
357         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
358         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
359         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
360
361         let mut node_0_stale_monitors_serialized = Vec::new();
362         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
363                 let mut writer = test_utils::TestVecWriter(Vec::new());
364                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
365                 node_0_stale_monitors_serialized.push(writer.0);
366         }
367
368         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
369
370         // Serialize the ChannelManager here, but the monitor we keep up-to-date
371         let nodes_0_serialized = nodes[0].node.encode();
372
373         route_payment(&nodes[0], &[&nodes[3]], 1000000);
374         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
375         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
376         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
377
378         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
379         // nodes[3])
380         let mut node_0_monitors_serialized = Vec::new();
381         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
382                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
383         }
384
385         logger = test_utils::TestLogger::new();
386         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
387         persister = test_utils::TestPersister::new();
388         let keys_manager = &chanmon_cfgs[0].keys_manager;
389         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
390         nodes[0].chain_monitor = &new_chain_monitor;
391
392
393         let mut node_0_stale_monitors = Vec::new();
394         for serialized in node_0_stale_monitors_serialized.iter() {
395                 let mut read = &serialized[..];
396                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
397                 assert!(read.is_empty());
398                 node_0_stale_monitors.push(monitor);
399         }
400
401         let mut node_0_monitors = Vec::new();
402         for serialized in node_0_monitors_serialized.iter() {
403                 let mut read = &serialized[..];
404                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
405                 assert!(read.is_empty());
406                 node_0_monitors.push(monitor);
407         }
408
409         let mut nodes_0_read = &nodes_0_serialized[..];
410         if let Err(msgs::DecodeError::InvalidValue) =
411                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
412                 default_config: UserConfig::default(),
413                 entropy_source: keys_manager,
414                 node_signer: keys_manager,
415                 signer_provider: keys_manager,
416                 fee_estimator: &fee_estimator,
417                 router: &nodes[0].router,
418                 chain_monitor: nodes[0].chain_monitor,
419                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
420                 logger: &logger,
421                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
422         }) { } else {
423                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
424         };
425
426         let mut nodes_0_read = &nodes_0_serialized[..];
427         let (_, nodes_0_deserialized_tmp) =
428                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
429                 default_config: UserConfig::default(),
430                 entropy_source: keys_manager,
431                 node_signer: keys_manager,
432                 signer_provider: keys_manager,
433                 fee_estimator: &fee_estimator,
434                 router: nodes[0].router,
435                 chain_monitor: nodes[0].chain_monitor,
436                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
437                 logger: &logger,
438                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
439         }).unwrap();
440         nodes_0_deserialized = nodes_0_deserialized_tmp;
441         assert!(nodes_0_read.is_empty());
442
443         for monitor in node_0_monitors.drain(..) {
444                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
445                         ChannelMonitorUpdateStatus::Completed);
446                 check_added_monitors!(nodes[0], 1);
447         }
448         nodes[0].node = &nodes_0_deserialized;
449
450         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
451         { // Channel close should result in a commitment tx
452                 nodes[0].node.timer_tick_occurred();
453                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
454                 assert_eq!(txn.len(), 1);
455                 check_spends!(txn[0], funding_tx);
456                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
457         }
458         check_added_monitors!(nodes[0], 1);
459
460         // nodes[1] and nodes[2] have no lost state with nodes[0]...
461         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
462         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
463         //... and we can even still claim the payment!
464         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
465
466         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
467                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
468         }, true).unwrap();
469         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
470         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
471                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
472         }, false).unwrap();
473         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
474         let mut found_err = false;
475         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
476                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
477                         match action {
478                                 &ErrorAction::SendErrorMessage { ref msg } => {
479                                         assert_eq!(msg.channel_id, channel_id);
480                                         assert!(!found_err);
481                                         found_err = true;
482                                 },
483                                 _ => panic!("Unexpected event!"),
484                         }
485                 }
486         }
487         assert!(found_err);
488 }
489
490 fn do_test_data_loss_protect(reconnect_panicing: bool) {
491         // When we get a data_loss_protect proving we're behind, we immediately panic as the
492         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
493         // panic message informs the user they should force-close without broadcasting, which is tested
494         // if `reconnect_panicing` is not set.
495         let mut chanmon_cfgs = create_chanmon_cfgs(2);
496         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
497         // during signing due to revoked tx
498         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
499         let persister;
500         let new_chain_monitor;
501         let nodes_0_deserialized;
502         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
503         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
504         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
505
506         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
507
508         // Cache node A state before any channel update
509         let previous_node_state = nodes[0].node.encode();
510         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
511
512         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
513         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
514
515         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
516         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
517
518         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
519
520         if reconnect_panicing {
521                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
522                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
523                 }, true).unwrap();
524                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
525                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
526                 }, false).unwrap();
527
528                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
529
530                 // Check we close channel detecting A is fallen-behind
531                 // Check that we sent the warning message when we detected that A has fallen behind,
532                 // and give the possibility for A to recover from the warning.
533                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
534                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
535                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
536
537                 {
538                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
539                         // The node B should not broadcast the transaction to force close the channel!
540                         assert!(node_txn.is_empty());
541                 }
542
543                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
544                 // Check A panics upon seeing proof it has fallen behind.
545                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
546                 return; // By this point we should have panic'ed!
547         }
548
549         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
550         check_added_monitors!(nodes[0], 1);
551         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
552         {
553                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
554                 assert_eq!(node_txn.len(), 0);
555         }
556
557         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
558                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
559                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
560                         match action {
561                                 &ErrorAction::SendErrorMessage { ref msg } => {
562                                         assert_eq!(msg.data, "Channel force-closed");
563                                 },
564                                 _ => panic!("Unexpected event!"),
565                         }
566                 } else {
567                         panic!("Unexpected event {:?}", msg)
568                 }
569         }
570
571         // after the warning message sent by B, we should not able to
572         // use the channel, or reconnect with success to the channel.
573         assert!(nodes[0].node.list_usable_channels().is_empty());
574         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
575                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
576         }, true).unwrap();
577         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
578                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
579         }, false).unwrap();
580         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
581
582         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
583         let mut err_msgs_0 = Vec::with_capacity(1);
584         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
585                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
586                         match action {
587                                 &ErrorAction::SendErrorMessage { ref msg } => {
588                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
589                                         err_msgs_0.push(msg.clone());
590                                 },
591                                 _ => panic!("Unexpected event!"),
592                         }
593                 } else {
594                         panic!("Unexpected event!");
595                 }
596         }
597         assert_eq!(err_msgs_0.len(), 1);
598         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
599         assert!(nodes[1].node.list_usable_channels().is_empty());
600         check_added_monitors!(nodes[1], 1);
601         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
602         check_closed_broadcast!(nodes[1], false);
603 }
604
605 #[test]
606 #[should_panic]
607 fn test_data_loss_protect_showing_stale_state_panics() {
608         do_test_data_loss_protect(true);
609 }
610
611 #[test]
612 fn test_force_close_without_broadcast() {
613         do_test_data_loss_protect(false);
614 }
615
616 #[test]
617 fn test_forwardable_regen() {
618         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
619         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
620         // HTLCs.
621         // We test it for both payment receipt and payment forwarding.
622
623         let chanmon_cfgs = create_chanmon_cfgs(3);
624         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
625         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
626         let persister: test_utils::TestPersister;
627         let new_chain_monitor: test_utils::TestChainMonitor;
628         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
629         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
630         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
631         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
632
633         // First send a payment to nodes[1]
634         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
635         nodes[0].node.send_payment_with_route(&route, payment_hash,
636                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
637         check_added_monitors!(nodes[0], 1);
638
639         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
640         assert_eq!(events.len(), 1);
641         let payment_event = SendEvent::from_event(events.pop().unwrap());
642         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
643         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
644
645         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
646
647         // Next send a payment which is forwarded by nodes[1]
648         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
649         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
650                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
651         check_added_monitors!(nodes[0], 1);
652
653         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
654         assert_eq!(events.len(), 1);
655         let payment_event = SendEvent::from_event(events.pop().unwrap());
656         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
657         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
658
659         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
660         // generated
661         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
662
663         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
664         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
665         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
666
667         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
668         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
669         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
670
671         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
672         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
673         // the commitment state.
674         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
675         reconnect_args.send_channel_ready = (true, true);
676         reconnect_nodes(reconnect_args);
677
678         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
679
680         expect_pending_htlcs_forwardable!(nodes[1]);
681         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
682         check_added_monitors!(nodes[1], 1);
683
684         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
685         assert_eq!(events.len(), 1);
686         let payment_event = SendEvent::from_event(events.pop().unwrap());
687         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
688         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
689         expect_pending_htlcs_forwardable!(nodes[2]);
690         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
691
692         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
693         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
694 }
695
696 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
697         // Test what happens if a node receives an MPP payment, claims it, but crashes before
698         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
699         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
700         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
701         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
702         // not have the preimage tied to the still-pending HTLC.
703         //
704         // To get to the correct state, on startup we should propagate the preimage to the
705         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
706         // receiving the preimage without a state update.
707         //
708         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
709         // definitely claimed.
710         let chanmon_cfgs = create_chanmon_cfgs(4);
711         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
712         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
713
714         let persister: test_utils::TestPersister;
715         let new_chain_monitor: test_utils::TestChainMonitor;
716         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
717
718         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
719
720         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
721         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
722         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
723         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
724
725         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
726         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
727         assert_eq!(route.paths.len(), 2);
728         route.paths.sort_by(|path_a, _| {
729                 // Sort the path so that the path through nodes[1] comes first
730                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
731                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
732         });
733
734         nodes[0].node.send_payment_with_route(&route, payment_hash,
735                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
736         check_added_monitors!(nodes[0], 2);
737
738         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
739         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
740         assert_eq!(send_events.len(), 2);
741         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
742         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
743         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
744         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
745
746         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
747         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
748         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
749         if !persist_both_monitors {
750                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
751                         if outpoint.to_channel_id() == chan_id_not_persisted {
752                                 assert!(original_monitor.0.is_empty());
753                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
754                         }
755                 }
756         }
757
758         let original_manager = nodes[3].node.encode();
759
760         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
761
762         nodes[3].node.claim_funds(payment_preimage);
763         check_added_monitors!(nodes[3], 2);
764         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
765
766         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
767         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
768         // with the old ChannelManager.
769         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
770         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
771                 if outpoint.to_channel_id() == chan_id_persisted {
772                         assert!(updated_monitor.0.is_empty());
773                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
774                 }
775         }
776         // If `persist_both_monitors` is set, get the second monitor here as well
777         if persist_both_monitors {
778                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
779                         if outpoint.to_channel_id() == chan_id_not_persisted {
780                                 assert!(original_monitor.0.is_empty());
781                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
782                         }
783                 }
784         }
785
786         // Now restart nodes[3].
787         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
788
789         // On startup the preimage should have been copied into the non-persisted monitor:
790         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
791         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
792
793         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
794         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
795
796         // During deserialization, we should have closed one channel and broadcast its latest
797         // commitment transaction. We should also still have the original PaymentClaimable event we
798         // never finished processing.
799         let events = nodes[3].node.get_and_clear_pending_events();
800         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
801         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
802         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
803         if persist_both_monitors {
804                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
805                 check_added_monitors(&nodes[3], 2);
806         } else {
807                 check_added_monitors(&nodes[3], 1);
808         }
809
810         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
811         // ChannelManager prior to handling the original one.
812         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
813                 events[if persist_both_monitors { 3 } else { 2 }]
814         {
815                 assert_eq!(payment_hash, our_payment_hash);
816         } else { panic!(); }
817
818         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
819         if !persist_both_monitors {
820                 // If one of the two channels is still live, reveal the payment preimage over it.
821
822                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
823                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
824                 }, true).unwrap();
825                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
826                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
827                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
828                 }, false).unwrap();
829                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
830
831                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
832                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
833                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
834
835                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
836
837                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
838                 // claim should fly.
839                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
840                 check_added_monitors!(nodes[3], 1);
841                 assert_eq!(ds_msgs.len(), 2);
842                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
843
844                 let cs_updates = match ds_msgs[1] {
845                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
846                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
847                                 check_added_monitors!(nodes[2], 1);
848                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
849                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
850                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
851                                 cs_updates
852                         }
853                         _ => panic!(),
854                 };
855
856                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
857                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
858                 expect_payment_sent!(nodes[0], payment_preimage);
859         }
860 }
861
862 #[test]
863 fn test_partial_claim_before_restart() {
864         do_test_partial_claim_before_restart(false);
865         do_test_partial_claim_before_restart(true);
866 }
867
868 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
869         if !use_cs_commitment { assert!(!claim_htlc); }
870         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
871         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
872         // it back until the ChannelMonitor decides the fate of the HTLC.
873         // This was never an issue, but it may be easy to regress here going forward.
874         let chanmon_cfgs = create_chanmon_cfgs(3);
875         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
876         let mut intercept_forwards_config = test_default_channel_config();
877         intercept_forwards_config.accept_intercept_htlcs = true;
878         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
879
880         let persister;
881         let new_chain_monitor;
882         let nodes_1_deserialized;
883
884         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
885
886         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
887         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
888
889         let intercept_scid = nodes[1].node.get_intercept_scid();
890
891         let (mut route, payment_hash, payment_preimage, payment_secret) =
892                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
893         if use_intercept {
894                 route.paths[0].hops[1].short_channel_id = intercept_scid;
895         }
896         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
897         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
898         nodes[0].node.send_payment_with_route(&route, payment_hash,
899                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
900         check_added_monitors!(nodes[0], 1);
901
902         let payment_event = SendEvent::from_node(&nodes[0]);
903         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
904         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
905
906         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
907         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
908         // present when we serialized.
909         let node_encoded = nodes[1].node.encode();
910
911         let mut intercept_id = None;
912         let mut expected_outbound_amount_msat = None;
913         if use_intercept {
914                 let events = nodes[1].node.get_and_clear_pending_events();
915                 assert_eq!(events.len(), 1);
916                 match events[0] {
917                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
918                                 intercept_id = Some(ev_id);
919                                 expected_outbound_amount_msat = Some(ev_amt);
920                         },
921                         _ => panic!()
922                 }
923                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
924                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
925         }
926
927         expect_pending_htlcs_forwardable!(nodes[1]);
928
929         let payment_event = SendEvent::from_node(&nodes[1]);
930         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
931         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
932         check_added_monitors!(nodes[2], 1);
933
934         if claim_htlc {
935                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
936                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
937         }
938         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
939
940         let _ = nodes[2].node.get_and_clear_pending_msg_events();
941
942         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
943         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
944         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
945
946         check_added_monitors!(nodes[2], 1);
947         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
948         check_closed_broadcast!(nodes[2], true);
949
950         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
951         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
952         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
953
954         // Note that this checks that this is the only event on nodes[1], implying the
955         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
956         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
957
958         if use_intercept {
959                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
960                 // a intercept-doesn't-exist error.
961                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
962                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
963                 assert_eq!(forward_err, APIError::APIMisuseError {
964                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
965                 });
966         }
967
968         nodes[1].node.timer_tick_occurred();
969         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
970         assert_eq!(bs_commitment_tx.len(), 1);
971         check_added_monitors!(nodes[1], 1);
972
973         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
974         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
975
976         if use_cs_commitment {
977                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
978                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
979                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
980                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
981                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
982
983                 if claim_htlc {
984                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
985                 } else {
986                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
987                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
988                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
989                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
990                 }
991         } else {
992                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
993         }
994
995         if !claim_htlc {
996                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
997         } else {
998                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
999         }
1000         check_added_monitors!(nodes[1], 1);
1001
1002         let events = nodes[1].node.get_and_clear_pending_msg_events();
1003         assert_eq!(events.len(), 1);
1004         match &events[0] {
1005                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1006                         if claim_htlc {
1007                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1008                         } else {
1009                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1010                         }
1011                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1012                 },
1013                 _ => panic!("Unexpected event"),
1014         }
1015
1016         if claim_htlc {
1017                 expect_payment_sent!(nodes[0], payment_preimage);
1018         } else {
1019                 expect_payment_failed!(nodes[0], payment_hash, false);
1020         }
1021 }
1022
1023 #[test]
1024 fn forwarded_payment_no_manager_persistence() {
1025         do_forwarded_payment_no_manager_persistence(true, true, false);
1026         do_forwarded_payment_no_manager_persistence(true, false, false);
1027         do_forwarded_payment_no_manager_persistence(false, false, false);
1028 }
1029
1030 #[test]
1031 fn intercepted_payment_no_manager_persistence() {
1032         do_forwarded_payment_no_manager_persistence(true, true, true);
1033         do_forwarded_payment_no_manager_persistence(true, false, true);
1034         do_forwarded_payment_no_manager_persistence(false, false, true);
1035 }
1036
1037 #[test]
1038 fn removed_payment_no_manager_persistence() {
1039         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1040         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1041         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1042         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1043         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1044         let chanmon_cfgs = create_chanmon_cfgs(3);
1045         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1046         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1047
1048         let persister;
1049         let new_chain_monitor;
1050         let nodes_1_deserialized;
1051
1052         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1053
1054         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1055         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1056
1057         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1058
1059         let node_encoded = nodes[1].node.encode();
1060
1061         nodes[2].node.fail_htlc_backwards(&payment_hash);
1062         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1063         check_added_monitors!(nodes[2], 1);
1064         let events = nodes[2].node.get_and_clear_pending_msg_events();
1065         assert_eq!(events.len(), 1);
1066         match &events[0] {
1067                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1068                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1069                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1070                 },
1071                 _ => panic!("Unexpected event"),
1072         }
1073
1074         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1075         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1076         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1077
1078         match nodes[1].node.pop_pending_event().unwrap() {
1079                 Event::ChannelClosed { ref reason, .. } => {
1080                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1081                 },
1082                 _ => panic!("Unexpected event"),
1083         }
1084
1085         nodes[1].node.test_process_background_events();
1086         check_added_monitors(&nodes[1], 1);
1087
1088         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1089         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1090         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1091         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1092         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1093
1094         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1095         check_added_monitors!(nodes[1], 1);
1096         let events = nodes[1].node.get_and_clear_pending_msg_events();
1097         assert_eq!(events.len(), 1);
1098         match &events[0] {
1099                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1100                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1101                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1102                 },
1103                 _ => panic!("Unexpected event"),
1104         }
1105
1106         expect_payment_failed!(nodes[0], payment_hash, false);
1107 }