]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
Add an immediately-freeing `MonitorUpdateCompletionAction`.
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::{CLOSED_CHANNEL_UPDATE_ID, ChannelMonitor};
15 use crate::sign::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
19 use crate::ln::msgs;
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::util::test_channel_signer::TestChannelSigner;
22 use crate::util::test_utils;
23 use crate::util::errors::APIError;
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26 use crate::util::string::UntrustedString;
27
28 use bitcoin::hash_types::BlockHash;
29
30 use crate::prelude::*;
31 use core::default::Default;
32 use crate::sync::Mutex;
33
34 use crate::ln::functional_test_utils::*;
35
36 #[test]
37 fn test_funding_peer_disconnect() {
38         // Test that we can lock in our funding tx while disconnected
39         let chanmon_cfgs = create_chanmon_cfgs(2);
40         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
41         let persister;
42         let new_chain_monitor;
43
44         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
45         let nodes_0_deserialized;
46         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
47         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
48
49         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
50         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
51
52         confirm_transaction(&nodes[0], &tx);
53         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
54         assert!(events_1.is_empty());
55
56         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
57         reconnect_args.send_channel_ready.1 = true;
58         reconnect_nodes(reconnect_args);
59
60         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
61         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
62
63         confirm_transaction(&nodes[1], &tx);
64         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
65         assert!(events_2.is_empty());
66
67         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
68                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
69         }, true).unwrap();
70         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
71         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
72                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
73         }, false).unwrap();
74         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
75
76         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
77         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
78         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
79         assert_eq!(events_3.len(), 1);
80         let as_channel_ready = match events_3[0] {
81                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
82                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
83                         msg.clone()
84                 },
85                 _ => panic!("Unexpected event {:?}", events_3[0]),
86         };
87
88         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
89         // announcement_signatures as well as channel_update.
90         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
91         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
92         assert_eq!(events_4.len(), 3);
93         let chan_id;
94         let bs_channel_ready = match events_4[0] {
95                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
96                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
97                         chan_id = msg.channel_id;
98                         msg.clone()
99                 },
100                 _ => panic!("Unexpected event {:?}", events_4[0]),
101         };
102         let bs_announcement_sigs = match events_4[1] {
103                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
104                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
105                         msg.clone()
106                 },
107                 _ => panic!("Unexpected event {:?}", events_4[1]),
108         };
109         match events_4[2] {
110                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
111                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
112                 },
113                 _ => panic!("Unexpected event {:?}", events_4[2]),
114         }
115
116         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
117         // generates a duplicative private channel_update
118         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
119         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
120         assert_eq!(events_5.len(), 1);
121         match events_5[0] {
122                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
123                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
124                 },
125                 _ => panic!("Unexpected event {:?}", events_5[0]),
126         };
127
128         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
129         // announcement_signatures.
130         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
131         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
132         assert_eq!(events_6.len(), 1);
133         let as_announcement_sigs = match events_6[0] {
134                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
135                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
136                         msg.clone()
137                 },
138                 _ => panic!("Unexpected event {:?}", events_6[0]),
139         };
140         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
141         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
142
143         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
144         // broadcast the channel announcement globally, as well as re-send its (now-public)
145         // channel_update.
146         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
147         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
148         assert_eq!(events_7.len(), 1);
149         let (chan_announcement, as_update) = match events_7[0] {
150                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
151                         (msg.clone(), update_msg.clone().unwrap())
152                 },
153                 _ => panic!("Unexpected event {:?}", events_7[0]),
154         };
155
156         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
157         // same channel_announcement.
158         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
159         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
160         assert_eq!(events_8.len(), 1);
161         let bs_update = match events_8[0] {
162                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
163                         assert_eq!(*msg, chan_announcement);
164                         update_msg.clone().unwrap()
165                 },
166                 _ => panic!("Unexpected event {:?}", events_8[0]),
167         };
168
169         // Provide the channel announcement and public updates to the network graph
170         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
171         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
172         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
173
174         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
175         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
176         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
177
178         // Check that after deserialization and reconnection we can still generate an identical
179         // channel_announcement from the cached signatures.
180         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
181
182         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
183
184         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
185
186         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
187 }
188
189 #[test]
190 fn test_no_txn_manager_serialize_deserialize() {
191         let chanmon_cfgs = create_chanmon_cfgs(2);
192         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
193         let persister;
194         let new_chain_monitor;
195
196         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
197         let nodes_0_deserialized;
198         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
199
200         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
201
202         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
203
204         let chan_0_monitor_serialized =
205                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
206         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
207
208         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
209                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
210         }, true).unwrap();
211         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
212         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
213                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
214         }, false).unwrap();
215         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
216
217         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
218         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
219         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
220         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
221
222         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
223         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
224         for node in nodes.iter() {
225                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
226                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
227                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
228         }
229
230         send_payment(&nodes[0], &[&nodes[1]], 1000000);
231 }
232
233 #[test]
234 fn test_manager_serialize_deserialize_events() {
235         // This test makes sure the events field in ChannelManager survives de/serialization
236         let chanmon_cfgs = create_chanmon_cfgs(2);
237         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
238         let persister;
239         let new_chain_monitor;
240
241         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
242         let nodes_0_deserialized;
243         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
244
245         // Start creating a channel, but stop right before broadcasting the funding transaction
246         let channel_value = 100000;
247         let push_msat = 10001;
248         let node_a = nodes.remove(0);
249         let node_b = nodes.remove(0);
250         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
251         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
252         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
253
254         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
255
256         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
257         check_added_monitors!(node_a, 0);
258
259         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
260         {
261                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
262                 assert_eq!(added_monitors.len(), 1);
263                 assert_eq!(added_monitors[0].0, funding_output);
264                 added_monitors.clear();
265         }
266
267         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
268         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
269         {
270                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
271                 assert_eq!(added_monitors.len(), 1);
272                 assert_eq!(added_monitors[0].0, funding_output);
273                 added_monitors.clear();
274         }
275         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
276
277         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
278         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
279
280         nodes.push(node_a);
281         nodes.push(node_b);
282
283         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
284         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
285         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
286
287         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
288
289         // After deserializing, make sure the funding_transaction is still held by the channel manager
290         let events_4 = nodes[0].node.get_and_clear_pending_events();
291         assert_eq!(events_4.len(), 0);
292         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
293         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
294
295         // Make sure the channel is functioning as though the de/serialization never happened
296         assert_eq!(nodes[0].node.list_channels().len(), 1);
297
298         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
299                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
300         }, true).unwrap();
301         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
302         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
303                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
304         }, false).unwrap();
305         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
306
307         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
308         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
309         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
310         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
311
312         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
313         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
314         for node in nodes.iter() {
315                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
316                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
317                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
318         }
319
320         send_payment(&nodes[0], &[&nodes[1]], 1000000);
321 }
322
323 #[test]
324 fn test_simple_manager_serialize_deserialize() {
325         let chanmon_cfgs = create_chanmon_cfgs(2);
326         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
327         let persister;
328         let new_chain_monitor;
329
330         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
331         let nodes_0_deserialized;
332         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
333         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
334
335         let (our_payment_preimage, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
336         let (_, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
337
338         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
339
340         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
341         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
342
343         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
344
345         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
346         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
347 }
348
349 #[test]
350 fn test_manager_serialize_deserialize_inconsistent_monitor() {
351         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
352         let chanmon_cfgs = create_chanmon_cfgs(4);
353         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
354         let logger;
355         let fee_estimator;
356         let persister;
357         let new_chain_monitor;
358
359         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
360         let nodes_0_deserialized;
361         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
362
363         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
364         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
365         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
366
367         let mut node_0_stale_monitors_serialized = Vec::new();
368         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
369                 let mut writer = test_utils::TestVecWriter(Vec::new());
370                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
371                 node_0_stale_monitors_serialized.push(writer.0);
372         }
373
374         let (our_payment_preimage, ..) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
375
376         // Serialize the ChannelManager here, but the monitor we keep up-to-date
377         let nodes_0_serialized = nodes[0].node.encode();
378
379         route_payment(&nodes[0], &[&nodes[3]], 1000000);
380         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
381         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
382         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
383
384         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
385         // nodes[3])
386         let mut node_0_monitors_serialized = Vec::new();
387         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
388                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
389         }
390
391         logger = test_utils::TestLogger::new();
392         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
393         persister = test_utils::TestPersister::new();
394         let keys_manager = &chanmon_cfgs[0].keys_manager;
395         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
396         nodes[0].chain_monitor = &new_chain_monitor;
397
398
399         let mut node_0_stale_monitors = Vec::new();
400         for serialized in node_0_stale_monitors_serialized.iter() {
401                 let mut read = &serialized[..];
402                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
403                 assert!(read.is_empty());
404                 node_0_stale_monitors.push(monitor);
405         }
406
407         let mut node_0_monitors = Vec::new();
408         for serialized in node_0_monitors_serialized.iter() {
409                 let mut read = &serialized[..];
410                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
411                 assert!(read.is_empty());
412                 node_0_monitors.push(monitor);
413         }
414
415         let mut nodes_0_read = &nodes_0_serialized[..];
416         if let Err(msgs::DecodeError::InvalidValue) =
417                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
418                 default_config: UserConfig::default(),
419                 entropy_source: keys_manager,
420                 node_signer: keys_manager,
421                 signer_provider: keys_manager,
422                 fee_estimator: &fee_estimator,
423                 router: &nodes[0].router,
424                 chain_monitor: nodes[0].chain_monitor,
425                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
426                 logger: &logger,
427                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
428         }) { } else {
429                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
430         };
431
432         let mut nodes_0_read = &nodes_0_serialized[..];
433         let (_, nodes_0_deserialized_tmp) =
434                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
435                 default_config: UserConfig::default(),
436                 entropy_source: keys_manager,
437                 node_signer: keys_manager,
438                 signer_provider: keys_manager,
439                 fee_estimator: &fee_estimator,
440                 router: nodes[0].router,
441                 chain_monitor: nodes[0].chain_monitor,
442                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
443                 logger: &logger,
444                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
445         }).unwrap();
446         nodes_0_deserialized = nodes_0_deserialized_tmp;
447         assert!(nodes_0_read.is_empty());
448
449         for monitor in node_0_monitors.drain(..) {
450                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
451                         Ok(ChannelMonitorUpdateStatus::Completed));
452                 check_added_monitors!(nodes[0], 1);
453         }
454         nodes[0].node = &nodes_0_deserialized;
455
456         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[3].node.get_our_node_id()], 100000);
457         { // Channel close should result in a commitment tx
458                 nodes[0].node.timer_tick_occurred();
459                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
460                 assert_eq!(txn.len(), 1);
461                 check_spends!(txn[0], funding_tx);
462                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
463         }
464         check_added_monitors!(nodes[0], 1);
465
466         // nodes[1] and nodes[2] have no lost state with nodes[0]...
467         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
468         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
469         //... and we can even still claim the payment!
470         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
471
472         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
473                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
474         }, true).unwrap();
475         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
476         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
477                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
478         }, false).unwrap();
479         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
480         let mut found_err = false;
481         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
482                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
483                         match action {
484                                 &ErrorAction::SendErrorMessage { ref msg } => {
485                                         assert_eq!(msg.channel_id, channel_id);
486                                         assert!(!found_err);
487                                         found_err = true;
488                                 },
489                                 _ => panic!("Unexpected event!"),
490                         }
491                 }
492         }
493         assert!(found_err);
494 }
495
496 fn do_test_data_loss_protect(reconnect_panicing: bool) {
497         // When we get a data_loss_protect proving we're behind, we immediately panic as the
498         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
499         // panic message informs the user they should force-close without broadcasting, which is tested
500         // if `reconnect_panicing` is not set.
501         let mut chanmon_cfgs = create_chanmon_cfgs(2);
502         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
503         // during signing due to revoked tx
504         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
505         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
506         let persister;
507         let new_chain_monitor;
508
509         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
510         let nodes_0_deserialized;
511
512         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
513
514         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
515
516         // Cache node A state before any channel update
517         let previous_node_state = nodes[0].node.encode();
518         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
519
520         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
521         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
522
523         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
524         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
525
526         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
527
528         if reconnect_panicing {
529                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
530                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
531                 }, true).unwrap();
532                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
533                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
534                 }, false).unwrap();
535
536                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
537
538                 // Check we close channel detecting A is fallen-behind
539                 // Check that we sent the warning message when we detected that A has fallen behind,
540                 // and give the possibility for A to recover from the warning.
541                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
542                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
543                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
544
545                 {
546                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
547                         // The node B should not broadcast the transaction to force close the channel!
548                         assert!(node_txn.is_empty());
549                 }
550
551                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
552                 // Check A panics upon seeing proof it has fallen behind.
553                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
554                 return; // By this point we should have panic'ed!
555         }
556
557         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
558         check_added_monitors!(nodes[0], 1);
559         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 1000000);
560         {
561                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
562                 assert_eq!(node_txn.len(), 0);
563         }
564
565         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
566                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
567                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
568                         match action {
569                                 &ErrorAction::SendErrorMessage { ref msg } => {
570                                         assert_eq!(msg.data, "Channel force-closed");
571                                 },
572                                 _ => panic!("Unexpected event!"),
573                         }
574                 } else {
575                         panic!("Unexpected event {:?}", msg)
576                 }
577         }
578
579         // after the warning message sent by B, we should not able to
580         // use the channel, or reconnect with success to the channel.
581         assert!(nodes[0].node.list_usable_channels().is_empty());
582         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
583                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
584         }, true).unwrap();
585         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
586                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
587         }, false).unwrap();
588         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
589
590         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
591         let mut err_msgs_0 = Vec::with_capacity(1);
592         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
593                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
594                         match action {
595                                 &ErrorAction::SendErrorMessage { ref msg } => {
596                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
597                                         err_msgs_0.push(msg.clone());
598                                 },
599                                 _ => panic!("Unexpected event!"),
600                         }
601                 } else {
602                         panic!("Unexpected event!");
603                 }
604         }
605         assert_eq!(err_msgs_0.len(), 1);
606         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
607         assert!(nodes[1].node.list_usable_channels().is_empty());
608         check_added_monitors!(nodes[1], 1);
609         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
610                 , [nodes[0].node.get_our_node_id()], 1000000);
611         check_closed_broadcast!(nodes[1], false);
612 }
613
614 #[test]
615 #[should_panic]
616 fn test_data_loss_protect_showing_stale_state_panics() {
617         do_test_data_loss_protect(true);
618 }
619
620 #[test]
621 fn test_force_close_without_broadcast() {
622         do_test_data_loss_protect(false);
623 }
624
625 #[test]
626 fn test_forwardable_regen() {
627         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
628         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
629         // HTLCs.
630         // We test it for both payment receipt and payment forwarding.
631
632         let chanmon_cfgs = create_chanmon_cfgs(3);
633         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
634         let persister;
635         let new_chain_monitor;
636         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
637         let nodes_1_deserialized;
638         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
639         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
640         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
641
642         // First send a payment to nodes[1]
643         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
644         nodes[0].node.send_payment_with_route(&route, payment_hash,
645                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
646         check_added_monitors!(nodes[0], 1);
647
648         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
649         assert_eq!(events.len(), 1);
650         let payment_event = SendEvent::from_event(events.pop().unwrap());
651         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
652         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
653
654         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
655
656         // Next send a payment which is forwarded by nodes[1]
657         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
658         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
659                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
660         check_added_monitors!(nodes[0], 1);
661
662         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
663         assert_eq!(events.len(), 1);
664         let payment_event = SendEvent::from_event(events.pop().unwrap());
665         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
666         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
667
668         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
669         // generated
670         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
671
672         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
673         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
674         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
675
676         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
677         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
678         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
679
680         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
681         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
682         // the commitment state.
683         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
684         reconnect_args.send_channel_ready = (true, true);
685         reconnect_nodes(reconnect_args);
686
687         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
688
689         expect_pending_htlcs_forwardable!(nodes[1]);
690         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
691         check_added_monitors!(nodes[1], 1);
692
693         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
694         assert_eq!(events.len(), 1);
695         let payment_event = SendEvent::from_event(events.pop().unwrap());
696         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
697         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
698         expect_pending_htlcs_forwardable!(nodes[2]);
699         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
700
701         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
702         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
703 }
704
705 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
706         // Test what happens if a node receives an MPP payment, claims it, but crashes before
707         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
708         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
709         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
710         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
711         // not have the preimage tied to the still-pending HTLC.
712         //
713         // To get to the correct state, on startup we should propagate the preimage to the
714         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
715         // receiving the preimage without a state update.
716         //
717         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
718         // definitely claimed.
719         let chanmon_cfgs = create_chanmon_cfgs(4);
720         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
721         let persister;
722         let new_chain_monitor;
723
724         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
725         let nodes_3_deserialized;
726
727         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
728
729         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
730         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
731         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
732         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
733
734         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
735         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
736         assert_eq!(route.paths.len(), 2);
737         route.paths.sort_by(|path_a, _| {
738                 // Sort the path so that the path through nodes[1] comes first
739                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
740                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
741         });
742
743         nodes[0].node.send_payment_with_route(&route, payment_hash,
744                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
745         check_added_monitors!(nodes[0], 2);
746
747         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
748         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
749         assert_eq!(send_events.len(), 2);
750         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
751         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
752         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
753         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
754
755         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
756         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
757         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
758         if !persist_both_monitors {
759                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
760                         if outpoint.to_channel_id() == chan_id_not_persisted {
761                                 assert!(original_monitor.0.is_empty());
762                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
763                         }
764                 }
765         }
766
767         let original_manager = nodes[3].node.encode();
768
769         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
770
771         nodes[3].node.claim_funds(payment_preimage);
772         check_added_monitors!(nodes[3], 2);
773         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
774
775         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
776         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
777         // with the old ChannelManager.
778         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
779         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
780                 if outpoint.to_channel_id() == chan_id_persisted {
781                         assert!(updated_monitor.0.is_empty());
782                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
783                 }
784         }
785         // If `persist_both_monitors` is set, get the second monitor here as well
786         if persist_both_monitors {
787                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
788                         if outpoint.to_channel_id() == chan_id_not_persisted {
789                                 assert!(original_monitor.0.is_empty());
790                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
791                         }
792                 }
793         }
794
795         // Now restart nodes[3].
796         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
797
798         // On startup the preimage should have been copied into the non-persisted monitor:
799         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
800         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
801
802         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
803         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
804
805         // During deserialization, we should have closed one channel and broadcast its latest
806         // commitment transaction. We should also still have the original PaymentClaimable event we
807         // never finished processing.
808         let events = nodes[3].node.get_and_clear_pending_events();
809         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
810         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
811         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
812         if persist_both_monitors {
813                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
814                 check_added_monitors(&nodes[3], 2);
815         } else {
816                 check_added_monitors(&nodes[3], 1);
817         }
818
819         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
820         // ChannelManager prior to handling the original one.
821         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
822                 events[if persist_both_monitors { 3 } else { 2 }]
823         {
824                 assert_eq!(payment_hash, our_payment_hash);
825         } else { panic!(); }
826
827         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
828         if !persist_both_monitors {
829                 // If one of the two channels is still live, reveal the payment preimage over it.
830
831                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
832                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
833                 }, true).unwrap();
834                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
835                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
836                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
837                 }, false).unwrap();
838                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
839
840                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
841                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
842                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
843
844                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
845
846                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
847                 // claim should fly.
848                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
849                 check_added_monitors!(nodes[3], 1);
850                 assert_eq!(ds_msgs.len(), 2);
851                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
852
853                 let cs_updates = match ds_msgs[1] {
854                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
855                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
856                                 check_added_monitors!(nodes[2], 1);
857                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
858                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
859                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
860                                 cs_updates
861                         }
862                         _ => panic!(),
863                 };
864
865                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
866                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
867                 expect_payment_sent!(nodes[0], payment_preimage);
868         }
869 }
870
871 #[test]
872 fn test_partial_claim_before_restart() {
873         do_test_partial_claim_before_restart(false);
874         do_test_partial_claim_before_restart(true);
875 }
876
877 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
878         if !use_cs_commitment { assert!(!claim_htlc); }
879         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
880         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
881         // it back until the ChannelMonitor decides the fate of the HTLC.
882         // This was never an issue, but it may be easy to regress here going forward.
883         let chanmon_cfgs = create_chanmon_cfgs(3);
884         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
885         let persister;
886         let new_chain_monitor;
887
888         let mut intercept_forwards_config = test_default_channel_config();
889         intercept_forwards_config.accept_intercept_htlcs = true;
890         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
891         let nodes_1_deserialized;
892
893         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
894
895         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
896         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
897
898         let intercept_scid = nodes[1].node.get_intercept_scid();
899
900         let (mut route, payment_hash, payment_preimage, payment_secret) =
901                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
902         if use_intercept {
903                 route.paths[0].hops[1].short_channel_id = intercept_scid;
904         }
905         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
906         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
907         nodes[0].node.send_payment_with_route(&route, payment_hash,
908                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
909         check_added_monitors!(nodes[0], 1);
910
911         let payment_event = SendEvent::from_node(&nodes[0]);
912         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
913         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
914
915         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
916         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
917         // present when we serialized.
918         let node_encoded = nodes[1].node.encode();
919
920         let mut intercept_id = None;
921         let mut expected_outbound_amount_msat = None;
922         if use_intercept {
923                 let events = nodes[1].node.get_and_clear_pending_events();
924                 assert_eq!(events.len(), 1);
925                 match events[0] {
926                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
927                                 intercept_id = Some(ev_id);
928                                 expected_outbound_amount_msat = Some(ev_amt);
929                         },
930                         _ => panic!()
931                 }
932                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
933                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
934         }
935
936         expect_pending_htlcs_forwardable!(nodes[1]);
937
938         let payment_event = SendEvent::from_node(&nodes[1]);
939         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
940         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
941         check_added_monitors!(nodes[2], 1);
942
943         if claim_htlc {
944                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
945                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
946         }
947         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
948
949         let _ = nodes[2].node.get_and_clear_pending_msg_events();
950
951         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
952         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
953         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
954
955         check_added_monitors!(nodes[2], 1);
956         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
957         check_closed_broadcast!(nodes[2], true);
958
959         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
960         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
961         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
962
963         // Note that this checks that this is the only event on nodes[1], implying the
964         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
965         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager, [nodes[2].node.get_our_node_id()], 100000);
966
967         if use_intercept {
968                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
969                 // a intercept-doesn't-exist error.
970                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
971                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
972                 assert_eq!(forward_err, APIError::APIMisuseError {
973                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
974                 });
975         }
976
977         nodes[1].node.timer_tick_occurred();
978         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
979         assert_eq!(bs_commitment_tx.len(), 1);
980         check_added_monitors!(nodes[1], 1);
981
982         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
983         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
984
985         if use_cs_commitment {
986                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
987                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
988                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
989                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
990                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
991
992                 if claim_htlc {
993                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
994                 } else {
995                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
996                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
997                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
998                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
999                 }
1000         } else {
1001                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
1002         }
1003
1004         if !claim_htlc {
1005                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1006         } else {
1007                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
1008         }
1009         check_added_monitors!(nodes[1], 1);
1010
1011         let events = nodes[1].node.get_and_clear_pending_msg_events();
1012         assert_eq!(events.len(), 1);
1013         match &events[0] {
1014                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1015                         if claim_htlc {
1016                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1017                         } else {
1018                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1019                         }
1020                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1021                 },
1022                 _ => panic!("Unexpected event"),
1023         }
1024
1025         if claim_htlc {
1026                 expect_payment_sent!(nodes[0], payment_preimage);
1027         } else {
1028                 expect_payment_failed!(nodes[0], payment_hash, false);
1029         }
1030 }
1031
1032 #[test]
1033 fn forwarded_payment_no_manager_persistence() {
1034         do_forwarded_payment_no_manager_persistence(true, true, false);
1035         do_forwarded_payment_no_manager_persistence(true, false, false);
1036         do_forwarded_payment_no_manager_persistence(false, false, false);
1037 }
1038
1039 #[test]
1040 fn intercepted_payment_no_manager_persistence() {
1041         do_forwarded_payment_no_manager_persistence(true, true, true);
1042         do_forwarded_payment_no_manager_persistence(true, false, true);
1043         do_forwarded_payment_no_manager_persistence(false, false, true);
1044 }
1045
1046 #[test]
1047 fn removed_payment_no_manager_persistence() {
1048         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1049         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1050         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1051         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1052         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1053         let chanmon_cfgs = create_chanmon_cfgs(3);
1054         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1055         let persister;
1056         let new_chain_monitor;
1057
1058         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1059         let nodes_1_deserialized;
1060
1061         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1062
1063         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1064         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1065
1066         let (_, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1067
1068         let node_encoded = nodes[1].node.encode();
1069
1070         nodes[2].node.fail_htlc_backwards(&payment_hash);
1071         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1072         check_added_monitors!(nodes[2], 1);
1073         let events = nodes[2].node.get_and_clear_pending_msg_events();
1074         assert_eq!(events.len(), 1);
1075         match &events[0] {
1076                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1077                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1078                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1079                 },
1080                 _ => panic!("Unexpected event"),
1081         }
1082
1083         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1084         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1085         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1086
1087         match nodes[1].node.pop_pending_event().unwrap() {
1088                 Event::ChannelClosed { ref reason, .. } => {
1089                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1090                 },
1091                 _ => panic!("Unexpected event"),
1092         }
1093
1094         nodes[1].node.test_process_background_events();
1095         check_added_monitors(&nodes[1], 1);
1096
1097         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1098         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1099         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1100         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1101         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1102
1103         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1104         check_added_monitors!(nodes[1], 1);
1105         let events = nodes[1].node.get_and_clear_pending_msg_events();
1106         assert_eq!(events.len(), 1);
1107         match &events[0] {
1108                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1109                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1110                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1111                 },
1112                 _ => panic!("Unexpected event"),
1113         }
1114
1115         expect_payment_failed!(nodes[0], payment_hash, false);
1116 }
1117
1118 #[test]
1119 fn test_reload_partial_funding_batch() {
1120         let chanmon_cfgs = create_chanmon_cfgs(3);
1121         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1122         let new_persister;
1123         let new_chain_monitor;
1124
1125         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1126         let new_channel_manager;
1127         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1128
1129         // Initiate channel opening and create the batch channel funding transaction.
1130         let (tx, funding_created_msgs) = create_batch_channel_funding(&nodes[0], &[
1131                 (&nodes[1], 100_000, 0, 42, None),
1132                 (&nodes[2], 200_000, 0, 43, None),
1133         ]);
1134
1135         // Go through the funding_created and funding_signed flow with node 1.
1136         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msgs[0]);
1137         check_added_monitors(&nodes[1], 1);
1138         expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
1139
1140         // The monitor is persisted when receiving funding_signed.
1141         let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
1142         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
1143         check_added_monitors(&nodes[0], 1);
1144
1145         // The transaction should not have been broadcast before all channels are ready.
1146         assert_eq!(nodes[0].tx_broadcaster.txn_broadcast().len(), 0);
1147
1148         // Reload the node while a subset of the channels in the funding batch have persisted monitors.
1149         let channel_id_1 = OutPoint { txid: tx.txid(), index: 0 }.to_channel_id();
1150         let node_encoded = nodes[0].node.encode();
1151         let channel_monitor_1_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1152         reload_node!(nodes[0], node_encoded, &[&channel_monitor_1_serialized], new_persister, new_chain_monitor, new_channel_manager);
1153
1154         // Process monitor events.
1155         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1156
1157         // The monitor should become closed.
1158         check_added_monitors(&nodes[0], 1);
1159         {
1160                 let mut monitor_updates = nodes[0].chain_monitor.monitor_updates.lock().unwrap();
1161                 let monitor_updates_1 = monitor_updates.get(&channel_id_1).unwrap();
1162                 assert_eq!(monitor_updates_1.len(), 1);
1163                 assert_eq!(monitor_updates_1[0].update_id, CLOSED_CHANNEL_UPDATE_ID);
1164         }
1165
1166         // The funding transaction should not have been broadcast, but we broadcast the force-close
1167         // transaction as part of closing the monitor.
1168         {
1169                 let broadcasted_txs = nodes[0].tx_broadcaster.txn_broadcast();
1170                 assert_eq!(broadcasted_txs.len(), 1);
1171                 assert!(broadcasted_txs[0].txid() != tx.txid());
1172                 assert_eq!(broadcasted_txs[0].input.len(), 1);
1173                 assert_eq!(broadcasted_txs[0].input[0].previous_output.txid, tx.txid());
1174         }
1175
1176         // Ensure the channels don't exist anymore.
1177         assert!(nodes[0].node.list_channels().is_empty());
1178 }