]> git.bitcoin.ninja Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs
Adapt intro node process_pending_htlcs test for non-intro nodes
[rust-lightning] / lightning / src / ln / reload_tests.rs
1 // This file is Copyright its original authors, visible in version control
2 // history.
3 //
4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
7 // You may not use this file except in accordance with one or both of these
8 // licenses.
9
10 //! Functional tests which test for correct behavior across node restarts.
11
12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
14 use crate::chain::channelmonitor::{CLOSED_CHANNEL_UPDATE_ID, ChannelMonitor};
15 use crate::sign::EntropySource;
16 use crate::chain::transaction::OutPoint;
17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
19 use crate::ln::{msgs, ChannelId};
20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
21 use crate::util::test_channel_signer::TestChannelSigner;
22 use crate::util::test_utils;
23 use crate::util::errors::APIError;
24 use crate::util::ser::{Writeable, ReadableArgs};
25 use crate::util::config::UserConfig;
26
27 use bitcoin::hash_types::BlockHash;
28
29 use crate::prelude::*;
30 use core::default::Default;
31 use crate::sync::Mutex;
32
33 use crate::ln::functional_test_utils::*;
34
35 #[test]
36 fn test_funding_peer_disconnect() {
37         // Test that we can lock in our funding tx while disconnected
38         let chanmon_cfgs = create_chanmon_cfgs(2);
39         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
40         let persister;
41         let new_chain_monitor;
42
43         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
44         let nodes_0_deserialized;
45         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
46         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
47
48         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
49         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
50
51         confirm_transaction(&nodes[0], &tx);
52         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
53         assert!(events_1.is_empty());
54
55         let mut reconnect_args = ReconnectArgs::new(&nodes[0], &nodes[1]);
56         reconnect_args.send_channel_ready.1 = true;
57         reconnect_nodes(reconnect_args);
58
59         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
60         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
61
62         confirm_transaction(&nodes[1], &tx);
63         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
64         assert!(events_2.is_empty());
65
66         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
67                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
68         }, true).unwrap();
69         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
70         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
71                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
72         }, false).unwrap();
73         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
74
75         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
76         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
77         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
78         assert_eq!(events_3.len(), 1);
79         let as_channel_ready = match events_3[0] {
80                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
81                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
82                         msg.clone()
83                 },
84                 _ => panic!("Unexpected event {:?}", events_3[0]),
85         };
86
87         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
88         // announcement_signatures as well as channel_update.
89         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
90         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
91         assert_eq!(events_4.len(), 3);
92         let chan_id;
93         let bs_channel_ready = match events_4[0] {
94                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
95                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
96                         chan_id = msg.channel_id;
97                         msg.clone()
98                 },
99                 _ => panic!("Unexpected event {:?}", events_4[0]),
100         };
101         let bs_announcement_sigs = match events_4[1] {
102                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
103                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
104                         msg.clone()
105                 },
106                 _ => panic!("Unexpected event {:?}", events_4[1]),
107         };
108         match events_4[2] {
109                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
110                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
111                 },
112                 _ => panic!("Unexpected event {:?}", events_4[2]),
113         }
114
115         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
116         // generates a duplicative private channel_update
117         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
118         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
119         assert_eq!(events_5.len(), 1);
120         match events_5[0] {
121                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
122                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
123                 },
124                 _ => panic!("Unexpected event {:?}", events_5[0]),
125         };
126
127         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
128         // announcement_signatures.
129         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
130         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
131         assert_eq!(events_6.len(), 1);
132         let as_announcement_sigs = match events_6[0] {
133                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
134                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
135                         msg.clone()
136                 },
137                 _ => panic!("Unexpected event {:?}", events_6[0]),
138         };
139         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
140         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
141
142         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
143         // broadcast the channel announcement globally, as well as re-send its (now-public)
144         // channel_update.
145         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
146         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
147         assert_eq!(events_7.len(), 1);
148         let (chan_announcement, as_update) = match events_7[0] {
149                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
150                         (msg.clone(), update_msg.clone().unwrap())
151                 },
152                 _ => panic!("Unexpected event {:?}", events_7[0]),
153         };
154
155         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
156         // same channel_announcement.
157         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
158         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
159         assert_eq!(events_8.len(), 1);
160         let bs_update = match events_8[0] {
161                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
162                         assert_eq!(*msg, chan_announcement);
163                         update_msg.clone().unwrap()
164                 },
165                 _ => panic!("Unexpected event {:?}", events_8[0]),
166         };
167
168         // Provide the channel announcement and public updates to the network graph
169         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
170         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
171         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
172
173         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
174         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
175         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
176
177         // Check that after deserialization and reconnection we can still generate an identical
178         // channel_announcement from the cached signatures.
179         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
180
181         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
182
183         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
184
185         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
186 }
187
188 #[test]
189 fn test_no_txn_manager_serialize_deserialize() {
190         let chanmon_cfgs = create_chanmon_cfgs(2);
191         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
192         let persister;
193         let new_chain_monitor;
194
195         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
196         let nodes_0_deserialized;
197         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
198
199         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
200
201         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
202
203         let chan_0_monitor_serialized =
204                 get_monitor!(nodes[0], ChannelId::v1_from_funding_outpoint(OutPoint { txid: tx.txid(), index: 0 })).encode();
205         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
206
207         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
208                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
209         }, true).unwrap();
210         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
211         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
212                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
213         }, false).unwrap();
214         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
215
216         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
217         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
218         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
219         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
220
221         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
222         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
223         for node in nodes.iter() {
224                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
225                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
226                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
227         }
228
229         send_payment(&nodes[0], &[&nodes[1]], 1000000);
230 }
231
232 #[test]
233 fn test_manager_serialize_deserialize_events() {
234         // This test makes sure the events field in ChannelManager survives de/serialization
235         let chanmon_cfgs = create_chanmon_cfgs(2);
236         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
237         let persister;
238         let new_chain_monitor;
239
240         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
241         let nodes_0_deserialized;
242         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
243
244         // Start creating a channel, but stop right before broadcasting the funding transaction
245         let channel_value = 100000;
246         let push_msat = 10001;
247         let node_a = nodes.remove(0);
248         let node_b = nodes.remove(0);
249         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None, None).unwrap();
250         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
251         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
252
253         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
254
255         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
256         check_added_monitors!(node_a, 0);
257
258         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
259         {
260                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
261                 assert_eq!(added_monitors.len(), 1);
262                 assert_eq!(added_monitors[0].0, funding_output);
263                 added_monitors.clear();
264         }
265
266         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
267         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
268         {
269                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
270                 assert_eq!(added_monitors.len(), 1);
271                 assert_eq!(added_monitors[0].0, funding_output);
272                 added_monitors.clear();
273         }
274         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
275
276         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
277         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
278
279         nodes.push(node_a);
280         nodes.push(node_b);
281
282         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
283         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
284         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
285
286         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
287
288         // After deserializing, make sure the funding_transaction is still held by the channel manager
289         let events_4 = nodes[0].node.get_and_clear_pending_events();
290         assert_eq!(events_4.len(), 0);
291         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
292         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
293
294         // Make sure the channel is functioning as though the de/serialization never happened
295         assert_eq!(nodes[0].node.list_channels().len(), 1);
296
297         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
298                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
299         }, true).unwrap();
300         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
301         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
302                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
303         }, false).unwrap();
304         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
305
306         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
307         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
308         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
309         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
310
311         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
312         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
313         for node in nodes.iter() {
314                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
315                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
316                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
317         }
318
319         send_payment(&nodes[0], &[&nodes[1]], 1000000);
320 }
321
322 #[test]
323 fn test_simple_manager_serialize_deserialize() {
324         let chanmon_cfgs = create_chanmon_cfgs(2);
325         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
326         let persister;
327         let new_chain_monitor;
328
329         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
330         let nodes_0_deserialized;
331         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
332         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
333
334         let (our_payment_preimage, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
335         let (_, our_payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
336
337         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
338
339         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
340         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
341
342         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
343
344         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
345         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
346 }
347
348 #[test]
349 fn test_manager_serialize_deserialize_inconsistent_monitor() {
350         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
351         let chanmon_cfgs = create_chanmon_cfgs(4);
352         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
353         let logger;
354         let fee_estimator;
355         let persister;
356         let new_chain_monitor;
357
358         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
359         let nodes_0_deserialized;
360         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
361
362         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
363         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
364         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
365
366         let mut node_0_stale_monitors_serialized = Vec::new();
367         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
368                 let mut writer = test_utils::TestVecWriter(Vec::new());
369                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
370                 node_0_stale_monitors_serialized.push(writer.0);
371         }
372
373         let (our_payment_preimage, ..) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
374
375         // Serialize the ChannelManager here, but the monitor we keep up-to-date
376         let nodes_0_serialized = nodes[0].node.encode();
377
378         route_payment(&nodes[0], &[&nodes[3]], 1000000);
379         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
380         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
381         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
382
383         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
384         // nodes[3])
385         let mut node_0_monitors_serialized = Vec::new();
386         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
387                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
388         }
389
390         logger = test_utils::TestLogger::new();
391         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
392         persister = test_utils::TestPersister::new();
393         let keys_manager = &chanmon_cfgs[0].keys_manager;
394         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
395         nodes[0].chain_monitor = &new_chain_monitor;
396
397
398         let mut node_0_stale_monitors = Vec::new();
399         for serialized in node_0_stale_monitors_serialized.iter() {
400                 let mut read = &serialized[..];
401                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
402                 assert!(read.is_empty());
403                 node_0_stale_monitors.push(monitor);
404         }
405
406         let mut node_0_monitors = Vec::new();
407         for serialized in node_0_monitors_serialized.iter() {
408                 let mut read = &serialized[..];
409                 let (_, monitor) = <(BlockHash, ChannelMonitor<TestChannelSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
410                 assert!(read.is_empty());
411                 node_0_monitors.push(monitor);
412         }
413
414         let mut nodes_0_read = &nodes_0_serialized[..];
415         if let Err(msgs::DecodeError::InvalidValue) =
416                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
417                 default_config: UserConfig::default(),
418                 entropy_source: keys_manager,
419                 node_signer: keys_manager,
420                 signer_provider: keys_manager,
421                 fee_estimator: &fee_estimator,
422                 router: &nodes[0].router,
423                 chain_monitor: nodes[0].chain_monitor,
424                 tx_broadcaster: nodes[0].tx_broadcaster,
425                 logger: &logger,
426                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
427         }) { } else {
428                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
429         };
430
431         let mut nodes_0_read = &nodes_0_serialized[..];
432         let (_, nodes_0_deserialized_tmp) =
433                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
434                 default_config: UserConfig::default(),
435                 entropy_source: keys_manager,
436                 node_signer: keys_manager,
437                 signer_provider: keys_manager,
438                 fee_estimator: &fee_estimator,
439                 router: nodes[0].router,
440                 chain_monitor: nodes[0].chain_monitor,
441                 tx_broadcaster: nodes[0].tx_broadcaster,
442                 logger: &logger,
443                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
444         }).unwrap();
445         nodes_0_deserialized = nodes_0_deserialized_tmp;
446         assert!(nodes_0_read.is_empty());
447
448         for monitor in node_0_monitors.drain(..) {
449                 let funding_outpoint = monitor.get_funding_txo().0;
450                 assert_eq!(nodes[0].chain_monitor.watch_channel(funding_outpoint, monitor),
451                         Ok(ChannelMonitorUpdateStatus::Completed));
452                 check_added_monitors!(nodes[0], 1);
453         }
454         nodes[0].node = &nodes_0_deserialized;
455
456         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager, [nodes[3].node.get_our_node_id()], 100000);
457         { // Channel close should result in a commitment tx
458                 nodes[0].node.timer_tick_occurred();
459                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
460                 assert_eq!(txn.len(), 1);
461                 check_spends!(txn[0], funding_tx);
462                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
463         }
464         check_added_monitors!(nodes[0], 1);
465
466         // nodes[1] and nodes[2] have no lost state with nodes[0]...
467         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
468         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[2]));
469         //... and we can even still claim the payment!
470         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
471
472         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
473                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
474         }, true).unwrap();
475         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
476         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
477                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
478         }, false).unwrap();
479         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
480         let mut found_err = false;
481         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
482                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
483                         match action {
484                                 &ErrorAction::SendErrorMessage { ref msg } => {
485                                         assert_eq!(msg.channel_id, channel_id);
486                                         assert!(!found_err);
487                                         found_err = true;
488                                 },
489                                 _ => panic!("Unexpected event!"),
490                         }
491                 }
492         }
493         assert!(found_err);
494 }
495
496 #[cfg(feature = "std")]
497 fn do_test_data_loss_protect(reconnect_panicing: bool, substantially_old: bool, not_stale: bool) {
498         use crate::routing::router::{RouteParameters, PaymentParameters};
499         use crate::ln::channelmanager::Retry;
500         use crate::util::string::UntrustedString;
501         // When we get a data_loss_protect proving we're behind, we immediately panic as the
502         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
503         // panic message informs the user they should force-close without broadcasting, which is tested
504         // if `reconnect_panicing` is not set.
505         let mut chanmon_cfgs = create_chanmon_cfgs(2);
506         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
507         // during signing due to revoked tx
508         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
509         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
510         let persister;
511         let new_chain_monitor;
512
513         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
514         let nodes_0_deserialized;
515
516         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
517
518         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
519
520         // Cache node A state before any channel update
521         let previous_node_state = nodes[0].node.encode();
522         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
523
524         assert!(!substantially_old || !not_stale, "substantially_old and not_stale doesn't make sense");
525         if not_stale || !substantially_old {
526                 // Previously, we'd only hit the data_loss_protect assertion if we had a state which
527                 // revoked at least two revocations ago, not the latest revocation. Here, we use
528                 // `not_stale` to test the boundary condition.
529                 let pay_params = PaymentParameters::for_keysend(nodes[1].node.get_our_node_id(), 100, false);
530                 let route_params = RouteParameters::from_payment_params_and_value(pay_params, 40000);
531                 nodes[0].node.send_spontaneous_payment_with_retry(None, RecipientOnionFields::spontaneous_empty(), PaymentId([0; 32]), route_params, Retry::Attempts(0)).unwrap();
532                 check_added_monitors(&nodes[0], 1);
533                 let update_add_commit = SendEvent::from_node(&nodes[0]);
534
535                 nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &update_add_commit.msgs[0]);
536                 nodes[1].node.handle_commitment_signed(&nodes[0].node.get_our_node_id(), &update_add_commit.commitment_msg);
537                 check_added_monitors(&nodes[1], 1);
538                 let (raa, cs) = get_revoke_commit_msgs(&nodes[1], &nodes[0].node.get_our_node_id());
539
540                 nodes[0].node.handle_revoke_and_ack(&nodes[1].node.get_our_node_id(), &raa);
541                 check_added_monitors(&nodes[0], 1);
542                 assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
543                 if !not_stale {
544                         nodes[0].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &cs);
545                         check_added_monitors(&nodes[0], 1);
546                         // A now revokes their original state, at which point reconnect should panic
547                         let raa = get_event_msg!(nodes[0], MessageSendEvent::SendRevokeAndACK, nodes[1].node.get_our_node_id());
548                         nodes[1].node.handle_revoke_and_ack(&nodes[0].node.get_our_node_id(), &raa);
549                         check_added_monitors(&nodes[1], 1);
550                         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
551                 }
552         } else {
553                 send_payment(&nodes[0], &[&nodes[1]], 8000000);
554                 send_payment(&nodes[0], &[&nodes[1]], 8000000);
555         }
556
557         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
558         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
559
560         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
561
562         if reconnect_panicing {
563                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
564                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
565                 }, true).unwrap();
566                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
567                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
568                 }, false).unwrap();
569
570                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
571
572                 // If A has fallen behind substantially, B should send it a message letting it know
573                 // that.
574                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
575                 let reestablish_msg;
576                 if substantially_old {
577                         let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction: 0 (received) vs 4 (expected)".to_owned();
578
579                         let warn_reestablish = nodes[1].node.get_and_clear_pending_msg_events();
580                         assert_eq!(warn_reestablish.len(), 2);
581                         match warn_reestablish[1] {
582                                 MessageSendEvent::HandleError { action: ErrorAction::SendWarningMessage { ref msg, .. }, .. } => {
583                                         assert_eq!(msg.data, warn_msg);
584                                 },
585                                 _ => panic!("Unexpected events: {:?}", warn_reestablish),
586                         }
587                         reestablish_msg = match &warn_reestablish[0] {
588                                 MessageSendEvent::SendChannelReestablish { msg, .. } => msg.clone(),
589                                 _ => panic!("Unexpected events: {:?}", warn_reestablish),
590                         };
591                 } else {
592                         let msgs = nodes[1].node.get_and_clear_pending_msg_events();
593                         assert!(msgs.len() >= 4);
594                         match msgs.last() {
595                                 Some(MessageSendEvent::SendChannelUpdate { .. }) => {},
596                                 _ => panic!("Unexpected events: {:?}", msgs),
597                         }
598                         assert!(msgs.iter().any(|msg| matches!(msg, MessageSendEvent::SendRevokeAndACK { .. })));
599                         assert!(msgs.iter().any(|msg| matches!(msg, MessageSendEvent::UpdateHTLCs { .. })));
600                         reestablish_msg = match &msgs[0] {
601                                 MessageSendEvent::SendChannelReestablish { msg, .. } => msg.clone(),
602                                 _ => panic!("Unexpected events: {:?}", msgs),
603                         };
604                 }
605
606                 {
607                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap();
608                         // The node B should never force-close the channel.
609                         assert!(node_txn.is_empty());
610                 }
611
612                 // Check A panics upon seeing proof it has fallen behind.
613                 let reconnect_res = std::panic::catch_unwind(|| {
614                         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_msg);
615                 });
616                 if not_stale {
617                         assert!(reconnect_res.is_ok());
618                         // At this point A gets confused because B expects a commitment state newer than A
619                         // has sent, but not a newer revocation secret, so A just (correctly) closes.
620                         check_closed_broadcast(&nodes[0], 1, true);
621                         check_added_monitors(&nodes[0], 1);
622                         check_closed_event!(nodes[0], 1, ClosureReason::ProcessingError {
623                                 err: "Peer attempted to reestablish channel with a future remote commitment transaction: 2 (received) vs 1 (expected)".to_owned()
624                         }, [nodes[1].node.get_our_node_id()], 1000000);
625                 } else {
626                         assert!(reconnect_res.is_err());
627                         // Skip the `Drop` handler for `Node`s as some may be in an invalid (panicked) state.
628                         std::mem::forget(nodes);
629                 }
630         } else {
631                 assert!(!not_stale, "We only care about the stale case when not testing panicking");
632
633                 nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
634                 check_added_monitors!(nodes[0], 1);
635                 check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 1000000);
636                 {
637                         let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
638                         assert_eq!(node_txn.len(), 0);
639                 }
640
641                 for msg in nodes[0].node.get_and_clear_pending_msg_events() {
642                         if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
643                         } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
644                                 match action {
645                                         &ErrorAction::DisconnectPeer { ref msg } => {
646                                                 assert_eq!(msg.as_ref().unwrap().data, "Channel force-closed");
647                                         },
648                                         _ => panic!("Unexpected event!"),
649                                 }
650                         } else {
651                                 panic!("Unexpected event {:?}", msg)
652                         }
653                 }
654
655                 // after the warning message sent by B, we should not able to
656                 // use the channel, or reconnect with success to the channel.
657                 assert!(nodes[0].node.list_usable_channels().is_empty());
658                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
659                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
660                 }, true).unwrap();
661                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
662                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
663                 }, false).unwrap();
664                 let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
665
666                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
667                 let mut err_msgs_0 = Vec::with_capacity(1);
668                 if let MessageSendEvent::HandleError { ref action, .. } = nodes[0].node.get_and_clear_pending_msg_events()[1] {
669                         match action {
670                                 &ErrorAction::SendErrorMessage { ref msg } => {
671                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
672                                         err_msgs_0.push(msg.clone());
673                                 },
674                                 _ => panic!("Unexpected event!"),
675                         }
676                 } else {
677                         panic!("Unexpected event!");
678                 }
679                 assert_eq!(err_msgs_0.len(), 1);
680                 nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
681                 assert!(nodes[1].node.list_usable_channels().is_empty());
682                 check_added_monitors!(nodes[1], 1);
683                 check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) }
684                         , [nodes[0].node.get_our_node_id()], 1000000);
685                 check_closed_broadcast!(nodes[1], false);
686         }
687 }
688
689 #[test]
690 #[cfg(feature = "std")]
691 fn test_data_loss_protect() {
692         do_test_data_loss_protect(true, false, true);
693         do_test_data_loss_protect(true, true, false);
694         do_test_data_loss_protect(true, false, false);
695         do_test_data_loss_protect(false, true, false);
696         do_test_data_loss_protect(false, false, false);
697 }
698
699 #[test]
700 fn test_forwardable_regen() {
701         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
702         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
703         // HTLCs.
704         // We test it for both payment receipt and payment forwarding.
705
706         let chanmon_cfgs = create_chanmon_cfgs(3);
707         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
708         let persister;
709         let new_chain_monitor;
710         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
711         let nodes_1_deserialized;
712         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
713         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
714         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
715
716         // First send a payment to nodes[1]
717         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
718         nodes[0].node.send_payment_with_route(&route, payment_hash,
719                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
720         check_added_monitors!(nodes[0], 1);
721
722         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
723         assert_eq!(events.len(), 1);
724         let payment_event = SendEvent::from_event(events.pop().unwrap());
725         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
726         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
727
728         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
729
730         // Next send a payment which is forwarded by nodes[1]
731         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
732         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
733                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
734         check_added_monitors!(nodes[0], 1);
735
736         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
737         assert_eq!(events.len(), 1);
738         let payment_event = SendEvent::from_event(events.pop().unwrap());
739         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
740         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
741
742         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
743         // generated
744         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
745
746         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
747         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
748         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
749
750         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
751         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
752         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
753
754         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
755         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
756         // the commitment state.
757         let mut reconnect_args = ReconnectArgs::new(&nodes[1], &nodes[2]);
758         reconnect_args.send_channel_ready = (true, true);
759         reconnect_nodes(reconnect_args);
760
761         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
762
763         expect_pending_htlcs_forwardable!(nodes[1]);
764         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
765         check_added_monitors!(nodes[1], 1);
766
767         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
768         assert_eq!(events.len(), 1);
769         let payment_event = SendEvent::from_event(events.pop().unwrap());
770         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
771         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
772         expect_pending_htlcs_forwardable!(nodes[2]);
773         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
774
775         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
776         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
777 }
778
779 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
780         // Test what happens if a node receives an MPP payment, claims it, but crashes before
781         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
782         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
783         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
784         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
785         // not have the preimage tied to the still-pending HTLC.
786         //
787         // To get to the correct state, on startup we should propagate the preimage to the
788         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
789         // receiving the preimage without a state update.
790         //
791         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
792         // definitely claimed.
793         let chanmon_cfgs = create_chanmon_cfgs(4);
794         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
795         let persister;
796         let new_chain_monitor;
797
798         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
799         let nodes_3_deserialized;
800
801         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
802
803         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
804         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
805         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
806         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
807
808         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
809         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
810         assert_eq!(route.paths.len(), 2);
811         route.paths.sort_by(|path_a, _| {
812                 // Sort the path so that the path through nodes[1] comes first
813                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
814                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
815         });
816
817         nodes[0].node.send_payment_with_route(&route, payment_hash,
818                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
819         check_added_monitors!(nodes[0], 2);
820
821         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
822         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
823         assert_eq!(send_events.len(), 2);
824         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
825         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
826         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None, false);
827         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None, false);
828
829         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
830         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
831         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
832         if !persist_both_monitors {
833                 for (outpoint, channel_id) in nodes[3].chain_monitor.chain_monitor.list_monitors() {
834                         if channel_id == chan_id_not_persisted {
835                                 assert!(original_monitor.0.is_empty());
836                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
837                         }
838                 }
839         }
840
841         let original_manager = nodes[3].node.encode();
842
843         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
844
845         nodes[3].node.claim_funds(payment_preimage);
846         check_added_monitors!(nodes[3], 2);
847         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
848
849         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
850         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
851         // with the old ChannelManager.
852         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
853         for (outpoint, channel_id) in nodes[3].chain_monitor.chain_monitor.list_monitors() {
854                 if channel_id == chan_id_persisted {
855                         assert!(updated_monitor.0.is_empty());
856                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
857                 }
858         }
859         // If `persist_both_monitors` is set, get the second monitor here as well
860         if persist_both_monitors {
861                 for (outpoint, channel_id) in nodes[3].chain_monitor.chain_monitor.list_monitors() {
862                         if channel_id == chan_id_not_persisted {
863                                 assert!(original_monitor.0.is_empty());
864                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
865                         }
866                 }
867         }
868
869         // Now restart nodes[3].
870         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
871
872         // On startup the preimage should have been copied into the non-persisted monitor:
873         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
874         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
875
876         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
877         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
878
879         // During deserialization, we should have closed one channel and broadcast its latest
880         // commitment transaction. We should also still have the original PaymentClaimable event we
881         // never finished processing.
882         let events = nodes[3].node.get_and_clear_pending_events();
883         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
884         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
885         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
886         if persist_both_monitors {
887                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
888                 check_added_monitors(&nodes[3], 2);
889         } else {
890                 check_added_monitors(&nodes[3], 1);
891         }
892
893         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
894         // ChannelManager prior to handling the original one.
895         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
896                 events[if persist_both_monitors { 3 } else { 2 }]
897         {
898                 assert_eq!(payment_hash, our_payment_hash);
899         } else { panic!(); }
900
901         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
902         if !persist_both_monitors {
903                 // If one of the two channels is still live, reveal the payment preimage over it.
904
905                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
906                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
907                 }, true).unwrap();
908                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
909                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
910                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
911                 }, false).unwrap();
912                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
913
914                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
915                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
916                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
917
918                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
919
920                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
921                 // claim should fly.
922                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
923                 check_added_monitors!(nodes[3], 1);
924                 assert_eq!(ds_msgs.len(), 2);
925                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
926
927                 let cs_updates = match ds_msgs[1] {
928                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
929                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
930                                 check_added_monitors!(nodes[2], 1);
931                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
932                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
933                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
934                                 cs_updates
935                         }
936                         _ => panic!(),
937                 };
938
939                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
940                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
941                 expect_payment_sent!(nodes[0], payment_preimage);
942         }
943 }
944
945 #[test]
946 fn test_partial_claim_before_restart() {
947         do_test_partial_claim_before_restart(false);
948         do_test_partial_claim_before_restart(true);
949 }
950
951 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
952         if !use_cs_commitment { assert!(!claim_htlc); }
953         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
954         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
955         // it back until the ChannelMonitor decides the fate of the HTLC.
956         // This was never an issue, but it may be easy to regress here going forward.
957         let chanmon_cfgs = create_chanmon_cfgs(3);
958         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
959         let persister;
960         let new_chain_monitor;
961
962         let mut intercept_forwards_config = test_default_channel_config();
963         intercept_forwards_config.accept_intercept_htlcs = true;
964         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
965         let nodes_1_deserialized;
966
967         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
968
969         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
970         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
971
972         let intercept_scid = nodes[1].node.get_intercept_scid();
973
974         let (mut route, payment_hash, payment_preimage, payment_secret) =
975                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
976         if use_intercept {
977                 route.paths[0].hops[1].short_channel_id = intercept_scid;
978         }
979         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
980         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
981         nodes[0].node.send_payment_with_route(&route, payment_hash,
982                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
983         check_added_monitors!(nodes[0], 1);
984
985         let payment_event = SendEvent::from_node(&nodes[0]);
986         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
987         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
988
989         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
990         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
991         // present when we serialized.
992         let node_encoded = nodes[1].node.encode();
993
994         let mut intercept_id = None;
995         let mut expected_outbound_amount_msat = None;
996         if use_intercept {
997                 let events = nodes[1].node.get_and_clear_pending_events();
998                 assert_eq!(events.len(), 1);
999                 match events[0] {
1000                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
1001                                 intercept_id = Some(ev_id);
1002                                 expected_outbound_amount_msat = Some(ev_amt);
1003                         },
1004                         _ => panic!()
1005                 }
1006                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
1007                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
1008         }
1009
1010         expect_pending_htlcs_forwardable!(nodes[1]);
1011
1012         let payment_event = SendEvent::from_node(&nodes[1]);
1013         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
1014         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
1015         check_added_monitors!(nodes[2], 1);
1016
1017         if claim_htlc {
1018                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
1019                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
1020         }
1021         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
1022
1023         let _ = nodes[2].node.get_and_clear_pending_msg_events();
1024
1025         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
1026         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1027         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
1028
1029         check_added_monitors!(nodes[2], 1);
1030         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed, [nodes[1].node.get_our_node_id()], 100000);
1031         check_closed_broadcast!(nodes[2], true);
1032
1033         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1034         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1035         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1036
1037         // Note that this checks that this is the only event on nodes[1], implying the
1038         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
1039         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager, [nodes[2].node.get_our_node_id()], 100000);
1040
1041         if use_intercept {
1042                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
1043                 // a intercept-doesn't-exist error.
1044                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
1045                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
1046                 assert_eq!(forward_err, APIError::APIMisuseError {
1047                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
1048                 });
1049         }
1050
1051         nodes[1].node.timer_tick_occurred();
1052         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
1053         assert_eq!(bs_commitment_tx.len(), 1);
1054         check_added_monitors!(nodes[1], 1);
1055
1056         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1057         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1058
1059         if use_cs_commitment {
1060                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
1061                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
1062                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
1063                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
1064                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
1065
1066                 if claim_htlc {
1067                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
1068                 } else {
1069                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
1070                         let mut txn = nodes[1].tx_broadcaster.txn_broadcast();
1071                         assert_eq!(txn.len(), if nodes[1].connect_style.borrow().updates_best_block_first() { 2 } else { 1 });
1072                         let bs_htlc_timeout_tx = txn.pop().unwrap();
1073                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx);
1074                 }
1075         } else {
1076                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
1077         }
1078
1079         if !claim_htlc {
1080                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1081         } else {
1082                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
1083         }
1084         check_added_monitors!(nodes[1], 1);
1085
1086         let events = nodes[1].node.get_and_clear_pending_msg_events();
1087         assert_eq!(events.len(), 1);
1088         match &events[0] {
1089                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1090                         if claim_htlc {
1091                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1092                         } else {
1093                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1094                         }
1095                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1096                 },
1097                 _ => panic!("Unexpected event"),
1098         }
1099
1100         if claim_htlc {
1101                 expect_payment_sent!(nodes[0], payment_preimage);
1102         } else {
1103                 expect_payment_failed!(nodes[0], payment_hash, false);
1104         }
1105 }
1106
1107 #[test]
1108 fn forwarded_payment_no_manager_persistence() {
1109         do_forwarded_payment_no_manager_persistence(true, true, false);
1110         do_forwarded_payment_no_manager_persistence(true, false, false);
1111         do_forwarded_payment_no_manager_persistence(false, false, false);
1112 }
1113
1114 #[test]
1115 fn intercepted_payment_no_manager_persistence() {
1116         do_forwarded_payment_no_manager_persistence(true, true, true);
1117         do_forwarded_payment_no_manager_persistence(true, false, true);
1118         do_forwarded_payment_no_manager_persistence(false, false, true);
1119 }
1120
1121 #[test]
1122 fn removed_payment_no_manager_persistence() {
1123         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1124         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1125         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1126         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1127         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1128         let chanmon_cfgs = create_chanmon_cfgs(3);
1129         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1130         let persister;
1131         let new_chain_monitor;
1132
1133         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1134         let nodes_1_deserialized;
1135
1136         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1137
1138         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1139         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1140
1141         let (_, payment_hash, ..) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1142
1143         let node_encoded = nodes[1].node.encode();
1144
1145         nodes[2].node.fail_htlc_backwards(&payment_hash);
1146         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1147         check_added_monitors!(nodes[2], 1);
1148         let events = nodes[2].node.get_and_clear_pending_msg_events();
1149         assert_eq!(events.len(), 1);
1150         match &events[0] {
1151                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1152                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1153                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1154                 },
1155                 _ => panic!("Unexpected event"),
1156         }
1157
1158         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1159         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1160         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1161
1162         match nodes[1].node.pop_pending_event().unwrap() {
1163                 Event::ChannelClosed { ref reason, .. } => {
1164                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1165                 },
1166                 _ => panic!("Unexpected event"),
1167         }
1168
1169         nodes[1].node.test_process_background_events();
1170         check_added_monitors(&nodes[1], 1);
1171
1172         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1173         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1174         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1175         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1176         reconnect_nodes(ReconnectArgs::new(&nodes[0], &nodes[1]));
1177
1178         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1179         check_added_monitors!(nodes[1], 1);
1180         let events = nodes[1].node.get_and_clear_pending_msg_events();
1181         assert_eq!(events.len(), 1);
1182         match &events[0] {
1183                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1184                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1185                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1186                 },
1187                 _ => panic!("Unexpected event"),
1188         }
1189
1190         expect_payment_failed!(nodes[0], payment_hash, false);
1191 }
1192
1193 #[test]
1194 fn test_reload_partial_funding_batch() {
1195         let chanmon_cfgs = create_chanmon_cfgs(3);
1196         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1197         let new_persister;
1198         let new_chain_monitor;
1199
1200         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1201         let new_channel_manager;
1202         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1203
1204         // Initiate channel opening and create the batch channel funding transaction.
1205         let (tx, funding_created_msgs) = create_batch_channel_funding(&nodes[0], &[
1206                 (&nodes[1], 100_000, 0, 42, None),
1207                 (&nodes[2], 200_000, 0, 43, None),
1208         ]);
1209
1210         // Go through the funding_created and funding_signed flow with node 1.
1211         nodes[1].node.handle_funding_created(&nodes[0].node.get_our_node_id(), &funding_created_msgs[0]);
1212         check_added_monitors(&nodes[1], 1);
1213         expect_channel_pending_event(&nodes[1], &nodes[0].node.get_our_node_id());
1214
1215         // The monitor is persisted when receiving funding_signed.
1216         let funding_signed_msg = get_event_msg!(nodes[1], MessageSendEvent::SendFundingSigned, nodes[0].node.get_our_node_id());
1217         nodes[0].node.handle_funding_signed(&nodes[1].node.get_our_node_id(), &funding_signed_msg);
1218         check_added_monitors(&nodes[0], 1);
1219
1220         // The transaction should not have been broadcast before all channels are ready.
1221         assert_eq!(nodes[0].tx_broadcaster.txn_broadcast().len(), 0);
1222
1223         // Reload the node while a subset of the channels in the funding batch have persisted monitors.
1224         let channel_id_1 = ChannelId::v1_from_funding_outpoint(OutPoint { txid: tx.txid(), index: 0 });
1225         let node_encoded = nodes[0].node.encode();
1226         let channel_monitor_1_serialized = get_monitor!(nodes[0], channel_id_1).encode();
1227         reload_node!(nodes[0], node_encoded, &[&channel_monitor_1_serialized], new_persister, new_chain_monitor, new_channel_manager);
1228
1229         // Process monitor events.
1230         assert!(nodes[0].node.get_and_clear_pending_events().is_empty());
1231
1232         // The monitor should become closed.
1233         check_added_monitors(&nodes[0], 1);
1234         {
1235                 let mut monitor_updates = nodes[0].chain_monitor.monitor_updates.lock().unwrap();
1236                 let monitor_updates_1 = monitor_updates.get(&channel_id_1).unwrap();
1237                 assert_eq!(monitor_updates_1.len(), 1);
1238                 assert_eq!(monitor_updates_1[0].update_id, CLOSED_CHANNEL_UPDATE_ID);
1239         }
1240
1241         // The funding transaction should not have been broadcast, but we broadcast the force-close
1242         // transaction as part of closing the monitor.
1243         {
1244                 let broadcasted_txs = nodes[0].tx_broadcaster.txn_broadcast();
1245                 assert_eq!(broadcasted_txs.len(), 1);
1246                 assert!(broadcasted_txs[0].txid() != tx.txid());
1247                 assert_eq!(broadcasted_txs[0].input.len(), 1);
1248                 assert_eq!(broadcasted_txs[0].input[0].previous_output.txid, tx.txid());
1249         }
1250
1251         // Ensure the channels don't exist anymore.
1252         assert!(nodes[0].node.list_channels().is_empty());
1253 }