_ Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test for correct behavior across node restarts.
  11
  12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
  13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
  14 use crate::chain::channelmonitor::ChannelMonitor;
  15 use crate::sign::EntropySource;
  16 use crate::chain::transaction::OutPoint;
  17 use crate::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  18 use crate::ln::channelmanager::{ChannelManager, ChannelManagerReadArgs, PaymentId, RecipientOnionFields};
  19 use crate::ln::msgs;
  20 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
  21 use crate::util::enforcing_trait_impls::EnforcingSigner;
  22 use crate::util::test_utils;
  23 use crate::util::errors::APIError;
  24 use crate::util::ser::{Writeable, ReadableArgs};
  25 use crate::util::config::UserConfig;
  26 use crate::util::string::UntrustedString;
  27
  28 use bitcoin::hash_types::BlockHash;
  29
  30 use crate::prelude::*;
  31 use core::default::Default;
  32 use crate::sync::Mutex;
  33
  34 use crate::ln::functional_test_utils::*;
  35
  36 #[test]
  37 fn test_funding_peer_disconnect() {
  38         // Test that we can lock in our funding tx while disconnected
  39         let chanmon_cfgs = create_chanmon_cfgs(2);
  40         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  41         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  42         let persister: test_utils::TestPersister;
  43         let new_chain_monitor: test_utils::TestChainMonitor;
  44         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
  45         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  46         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
  47
  48         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
  49         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
  50
  51         confirm_transaction(&nodes[0], &tx);
  52         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  53         assert!(events_1.is_empty());
  54
  55         reconnect_nodes(&nodes[0], &nodes[1], (false, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  56
  57         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
  58         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
  59
  60         confirm_transaction(&nodes[1], &tx);
  61         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
  62         assert!(events_2.is_empty());
  63
  64         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
  65                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
  66         }, true).unwrap();
  67         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
  68         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
  69                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
  70         }, false).unwrap();
  71         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
  72
  73         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
  74         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
  75         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
  76         assert_eq!(events_3.len(), 1);
  77         let as_channel_ready = match events_3[0] {
  78                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  79                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
  80                         msg.clone()
  81                 },
  82                 _ => panic!("Unexpected event {:?}", events_3[0]),
  83         };
  84
  85         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
  86         // announcement_signatures as well as channel_update.
  87         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
  88         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
  89         assert_eq!(events_4.len(), 3);
  90         let chan_id;
  91         let bs_channel_ready = match events_4[0] {
  92                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  93                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  94                         chan_id = msg.channel_id;
  95                         msg.clone()
  96                 },
  97                 _ => panic!("Unexpected event {:?}", events_4[0]),
  98         };
  99         let bs_announcement_sigs = match events_4[1] {
 100                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 101                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 102                         msg.clone()
 103                 },
 104                 _ => panic!("Unexpected event {:?}", events_4[1]),
 105         };
 106         match events_4[2] {
 107                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 108                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 109                 },
 110                 _ => panic!("Unexpected event {:?}", events_4[2]),
 111         }
 112
 113         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
 114         // generates a duplicative private channel_update
 115         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
 116         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
 117         assert_eq!(events_5.len(), 1);
 118         match events_5[0] {
 119                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 120                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 121                 },
 122                 _ => panic!("Unexpected event {:?}", events_5[0]),
 123         };
 124
 125         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
 126         // announcement_signatures.
 127         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
 128         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
 129         assert_eq!(events_6.len(), 1);
 130         let as_announcement_sigs = match events_6[0] {
 131                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 132                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 133                         msg.clone()
 134                 },
 135                 _ => panic!("Unexpected event {:?}", events_6[0]),
 136         };
 137         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
 138         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 139
 140         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
 141         // broadcast the channel announcement globally, as well as re-send its (now-public)
 142         // channel_update.
 143         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
 144         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
 145         assert_eq!(events_7.len(), 1);
 146         let (chan_announcement, as_update) = match events_7[0] {
 147                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 148                         (msg.clone(), update_msg.clone().unwrap())
 149                 },
 150                 _ => panic!("Unexpected event {:?}", events_7[0]),
 151         };
 152
 153         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
 154         // same channel_announcement.
 155         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
 156         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
 157         assert_eq!(events_8.len(), 1);
 158         let bs_update = match events_8[0] {
 159                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 160                         assert_eq!(*msg, chan_announcement);
 161                         update_msg.clone().unwrap()
 162                 },
 163                 _ => panic!("Unexpected event {:?}", events_8[0]),
 164         };
 165
 166         // Provide the channel announcement and public updates to the network graph
 167         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
 168         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
 169         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
 170
 171         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
 172         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
 173         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 174
 175         // Check that after deserialization and reconnection we can still generate an identical
 176         // channel_announcement from the cached signatures.
 177         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 178
 179         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 180
 181         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 182
 183         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 184 }
 185
 186 #[test]
 187 fn test_no_txn_manager_serialize_deserialize() {
 188         let chanmon_cfgs = create_chanmon_cfgs(2);
 189         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 190         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 191         let persister: test_utils::TestPersister;
 192         let new_chain_monitor: test_utils::TestChainMonitor;
 193         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 194         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 195
 196         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001);
 197
 198         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 199
 200         let chan_0_monitor_serialized =
 201                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
 202         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 203
 204         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 205                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 206         }, true).unwrap();
 207         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 208         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 209                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 210         }, false).unwrap();
 211         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 212
 213         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 214         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 215         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 216         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 217
 218         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 219         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 220         for node in nodes.iter() {
 221                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 222                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 223                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 224         }
 225
 226         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 227 }
 228
 229 #[test]
 230 fn test_manager_serialize_deserialize_events() {
 231         // This test makes sure the events field in ChannelManager survives de/serialization
 232         let chanmon_cfgs = create_chanmon_cfgs(2);
 233         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 234         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 235         let persister: test_utils::TestPersister;
 236         let new_chain_monitor: test_utils::TestChainMonitor;
 237         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 238         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 239
 240         // Start creating a channel, but stop right before broadcasting the funding transaction
 241         let channel_value = 100000;
 242         let push_msat = 10001;
 243         let node_a = nodes.remove(0);
 244         let node_b = nodes.remove(0);
 245         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
 246         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
 247         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
 248
 249         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
 250
 251         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
 252         check_added_monitors!(node_a, 0);
 253
 254         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
 255         {
 256                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
 257                 assert_eq!(added_monitors.len(), 1);
 258                 assert_eq!(added_monitors[0].0, funding_output);
 259                 added_monitors.clear();
 260         }
 261
 262         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
 263         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
 264         {
 265                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
 266                 assert_eq!(added_monitors.len(), 1);
 267                 assert_eq!(added_monitors[0].0, funding_output);
 268                 added_monitors.clear();
 269         }
 270         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
 271
 272         expect_channel_pending_event(&node_a, &node_b.node.get_our_node_id());
 273         expect_channel_pending_event(&node_b, &node_a.node.get_our_node_id());
 274
 275         nodes.push(node_a);
 276         nodes.push(node_b);
 277
 278         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
 279         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
 280         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 281
 282         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 283
 284         // After deserializing, make sure the funding_transaction is still held by the channel manager
 285         let events_4 = nodes[0].node.get_and_clear_pending_events();
 286         assert_eq!(events_4.len(), 0);
 287         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
 288         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
 289
 290         // Make sure the channel is functioning as though the de/serialization never happened
 291         assert_eq!(nodes[0].node.list_channels().len(), 1);
 292
 293         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 294                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 295         }, true).unwrap();
 296         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 297         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 298                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 299         }, false).unwrap();
 300         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 301
 302         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 303         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 304         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 305         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 306
 307         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 308         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 309         for node in nodes.iter() {
 310                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 311                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 312                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 313         }
 314
 315         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 316 }
 317
 318 #[test]
 319 fn test_simple_manager_serialize_deserialize() {
 320         let chanmon_cfgs = create_chanmon_cfgs(2);
 321         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 322         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 323         let persister: test_utils::TestPersister;
 324         let new_chain_monitor: test_utils::TestChainMonitor;
 325         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 326         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 327         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 328
 329         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 330         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 331
 332         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 333
 334         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 335         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 336
 337         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 338
 339         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
 340         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
 341 }
 342
 343 #[test]
 344 fn test_manager_serialize_deserialize_inconsistent_monitor() {
 345         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
 346         let chanmon_cfgs = create_chanmon_cfgs(4);
 347         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 348         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 349         let logger: test_utils::TestLogger;
 350         let fee_estimator: test_utils::TestFeeEstimator;
 351         let persister: test_utils::TestPersister;
 352         let new_chain_monitor: test_utils::TestChainMonitor;
 353         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 354         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 355         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 356         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0).2;
 357         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3);
 358
 359         let mut node_0_stale_monitors_serialized = Vec::new();
 360         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 361                 let mut writer = test_utils::TestVecWriter(Vec::new());
 362                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
 363                 node_0_stale_monitors_serialized.push(writer.0);
 364         }
 365
 366         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
 367
 368         // Serialize the ChannelManager here, but the monitor we keep up-to-date
 369         let nodes_0_serialized = nodes[0].node.encode();
 370
 371         route_payment(&nodes[0], &[&nodes[3]], 1000000);
 372         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 373         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 374         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 375
 376         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
 377         // nodes[3])
 378         let mut node_0_monitors_serialized = Vec::new();
 379         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 380                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
 381         }
 382
 383         logger = test_utils::TestLogger::new();
 384         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
 385         persister = test_utils::TestPersister::new();
 386         let keys_manager = &chanmon_cfgs[0].keys_manager;
 387         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster, &logger, &fee_estimator, &persister, keys_manager);
 388         nodes[0].chain_monitor = &new_chain_monitor;
 389
 390
 391         let mut node_0_stale_monitors = Vec::new();
 392         for serialized in node_0_stale_monitors_serialized.iter() {
 393                 let mut read = &serialized[..];
 394                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
 395                 assert!(read.is_empty());
 396                 node_0_stale_monitors.push(monitor);
 397         }
 398
 399         let mut node_0_monitors = Vec::new();
 400         for serialized in node_0_monitors_serialized.iter() {
 401                 let mut read = &serialized[..];
 402                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, (keys_manager, keys_manager)).unwrap();
 403                 assert!(read.is_empty());
 404                 node_0_monitors.push(monitor);
 405         }
 406
 407         let mut nodes_0_read = &nodes_0_serialized[..];
 408         if let Err(msgs::DecodeError::InvalidValue) =
 409                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 410                 default_config: UserConfig::default(),
 411                 entropy_source: keys_manager,
 412                 node_signer: keys_manager,
 413                 signer_provider: keys_manager,
 414                 fee_estimator: &fee_estimator,
 415                 router: &nodes[0].router,
 416                 chain_monitor: nodes[0].chain_monitor,
 417                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 418                 logger: &logger,
 419                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 420         }) { } else {
 421                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
 422         };
 423
 424         let mut nodes_0_read = &nodes_0_serialized[..];
 425         let (_, nodes_0_deserialized_tmp) =
 426                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 427                 default_config: UserConfig::default(),
 428                 entropy_source: keys_manager,
 429                 node_signer: keys_manager,
 430                 signer_provider: keys_manager,
 431                 fee_estimator: &fee_estimator,
 432                 router: nodes[0].router,
 433                 chain_monitor: nodes[0].chain_monitor,
 434                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 435                 logger: &logger,
 436                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 437         }).unwrap();
 438         nodes_0_deserialized = nodes_0_deserialized_tmp;
 439         assert!(nodes_0_read.is_empty());
 440
 441         for monitor in node_0_monitors.drain(..) {
 442                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
 443                         ChannelMonitorUpdateStatus::Completed);
 444                 check_added_monitors!(nodes[0], 1);
 445         }
 446         nodes[0].node = &nodes_0_deserialized;
 447
 448         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 449         { // Channel close should result in a commitment tx
 450                 nodes[0].node.timer_tick_occurred();
 451                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 452                 assert_eq!(txn.len(), 1);
 453                 check_spends!(txn[0], funding_tx);
 454                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
 455         }
 456         check_added_monitors!(nodes[0], 1);
 457
 458         // nodes[1] and nodes[2] have no lost state with nodes[0]...
 459         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 460         reconnect_nodes(&nodes[0], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 461         //... and we can even still claim the payment!
 462         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
 463
 464         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 465                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 466         }, true).unwrap();
 467         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
 468         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
 469                 features: nodes[3].node.init_features(), networks: None, remote_network_address: None
 470         }, false).unwrap();
 471         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
 472         let mut found_err = false;
 473         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
 474                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
 475                         match action {
 476                                 &ErrorAction::SendErrorMessage { ref msg } => {
 477                                         assert_eq!(msg.channel_id, channel_id);
 478                                         assert!(!found_err);
 479                                         found_err = true;
 480                                 },
 481                                 _ => panic!("Unexpected event!"),
 482                         }
 483                 }
 484         }
 485         assert!(found_err);
 486 }
 487
 488 fn do_test_data_loss_protect(reconnect_panicing: bool) {
 489         // When we get a data_loss_protect proving we're behind, we immediately panic as the
 490         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
 491         // panic message informs the user they should force-close without broadcasting, which is tested
 492         // if `reconnect_panicing` is not set.
 493         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 494         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
 495         // during signing due to revoked tx
 496         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
 497         let persister;
 498         let new_chain_monitor;
 499         let nodes_0_deserialized;
 500         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 501         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 502         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 503
 504         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000);
 505
 506         // Cache node A state before any channel update
 507         let previous_node_state = nodes[0].node.encode();
 508         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
 509
 510         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 511         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 512
 513         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 514         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id());
 515
 516         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
 517
 518         if reconnect_panicing {
 519                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 520                         features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 521                 }, true).unwrap();
 522                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 523                         features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 524                 }, false).unwrap();
 525
 526                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 527
 528                 // Check we close channel detecting A is fallen-behind
 529                 // Check that we sent the warning message when we detected that A has fallen behind,
 530                 // and give the possibility for A to recover from the warning.
 531                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 532                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
 533                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
 534
 535                 {
 536                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
 537                         // The node B should not broadcast the transaction to force close the channel!
 538                         assert!(node_txn.is_empty());
 539                 }
 540
 541                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 542                 // Check A panics upon seeing proof it has fallen behind.
 543                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
 544                 return; // By this point we should have panic'ed!
 545         }
 546
 547         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
 548         check_added_monitors!(nodes[0], 1);
 549         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 550         {
 551                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 552                 assert_eq!(node_txn.len(), 0);
 553         }
 554
 555         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 556                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
 557                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
 558                         match action {
 559                                 &ErrorAction::SendErrorMessage { ref msg } => {
 560                                         assert_eq!(msg.data, "Channel force-closed");
 561                                 },
 562                                 _ => panic!("Unexpected event!"),
 563                         }
 564                 } else {
 565                         panic!("Unexpected event {:?}", msg)
 566                 }
 567         }
 568
 569         // after the warning message sent by B, we should not able to
 570         // use the channel, or reconnect with success to the channel.
 571         assert!(nodes[0].node.list_usable_channels().is_empty());
 572         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init {
 573                 features: nodes[1].node.init_features(), networks: None, remote_network_address: None
 574         }, true).unwrap();
 575         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init {
 576                 features: nodes[0].node.init_features(), networks: None, remote_network_address: None
 577         }, false).unwrap();
 578         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 579
 580         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
 581         let mut err_msgs_0 = Vec::with_capacity(1);
 582         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 583                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
 584                         match action {
 585                                 &ErrorAction::SendErrorMessage { ref msg } => {
 586                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
 587                                         err_msgs_0.push(msg.clone());
 588                                 },
 589                                 _ => panic!("Unexpected event!"),
 590                         }
 591                 } else {
 592                         panic!("Unexpected event!");
 593                 }
 594         }
 595         assert_eq!(err_msgs_0.len(), 1);
 596         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
 597         assert!(nodes[1].node.list_usable_channels().is_empty());
 598         check_added_monitors!(nodes[1], 1);
 599         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: UntrustedString(format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id())) });
 600         check_closed_broadcast!(nodes[1], false);
 601 }
 602
 603 #[test]
 604 #[should_panic]
 605 fn test_data_loss_protect_showing_stale_state_panics() {
 606         do_test_data_loss_protect(true);
 607 }
 608
 609 #[test]
 610 fn test_force_close_without_broadcast() {
 611         do_test_data_loss_protect(false);
 612 }
 613
 614 #[test]
 615 fn test_forwardable_regen() {
 616         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
 617         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
 618         // HTLCs.
 619         // We test it for both payment receipt and payment forwarding.
 620
 621         let chanmon_cfgs = create_chanmon_cfgs(3);
 622         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 623         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 624         let persister: test_utils::TestPersister;
 625         let new_chain_monitor: test_utils::TestChainMonitor;
 626         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 627         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 628         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 629         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 630
 631         // First send a payment to nodes[1]
 632         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 633         nodes[0].node.send_payment_with_route(&route, payment_hash,
 634                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 635         check_added_monitors!(nodes[0], 1);
 636
 637         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 638         assert_eq!(events.len(), 1);
 639         let payment_event = SendEvent::from_event(events.pop().unwrap());
 640         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 641         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 642
 643         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
 644
 645         // Next send a payment which is forwarded by nodes[1]
 646         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
 647         nodes[0].node.send_payment_with_route(&route_2, payment_hash_2,
 648                 RecipientOnionFields::secret_only(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 649         check_added_monitors!(nodes[0], 1);
 650
 651         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 652         assert_eq!(events.len(), 1);
 653         let payment_event = SendEvent::from_event(events.pop().unwrap());
 654         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 655         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 656
 657         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
 658         // generated
 659         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 660
 661         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
 662         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 663         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 664
 665         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 666         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 667         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 668
 669         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 670         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
 671         // the commitment state.
 672         reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 673
 674         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 675
 676         expect_pending_htlcs_forwardable!(nodes[1]);
 677         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
 678         check_added_monitors!(nodes[1], 1);
 679
 680         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
 681         assert_eq!(events.len(), 1);
 682         let payment_event = SendEvent::from_event(events.pop().unwrap());
 683         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 684         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
 685         expect_pending_htlcs_forwardable!(nodes[2]);
 686         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
 687
 688         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 689         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
 690 }
 691
 692 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 693         // Test what happens if a node receives an MPP payment, claims it, but crashes before
 694         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
 695         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
 696         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
 697         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
 698         // not have the preimage tied to the still-pending HTLC.
 699         //
 700         // To get to the correct state, on startup we should propagate the preimage to the
 701         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
 702         // receiving the preimage without a state update.
 703         //
 704         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
 705         // definitely claimed.
 706         let chanmon_cfgs = create_chanmon_cfgs(4);
 707         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 708         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 709
 710         let persister: test_utils::TestPersister;
 711         let new_chain_monitor: test_utils::TestChainMonitor;
 712         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 713
 714         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 715
 716         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0);
 717         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0);
 718         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0).2;
 719         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0).2;
 720
 721         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
 722         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
 723         assert_eq!(route.paths.len(), 2);
 724         route.paths.sort_by(|path_a, _| {
 725                 // Sort the path so that the path through nodes[1] comes first
 726                 if path_a.hops[0].pubkey == nodes[1].node.get_our_node_id() {
 727                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
 728         });
 729
 730         nodes[0].node.send_payment_with_route(&route, payment_hash,
 731                 RecipientOnionFields::secret_only(payment_secret), PaymentId(payment_hash.0)).unwrap();
 732         check_added_monitors!(nodes[0], 2);
 733
 734         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
 735         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
 736         assert_eq!(send_events.len(), 2);
 737         let node_1_msgs = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &mut send_events);
 738         let node_2_msgs = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &mut send_events);
 739         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
 740         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
 741
 742         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
 743         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
 744         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
 745         if !persist_both_monitors {
 746                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 747                         if outpoint.to_channel_id() == chan_id_not_persisted {
 748                                 assert!(original_monitor.0.is_empty());
 749                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 750                         }
 751                 }
 752         }
 753
 754         let original_manager = nodes[3].node.encode();
 755
 756         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
 757
 758         nodes[3].node.claim_funds(payment_preimage);
 759         check_added_monitors!(nodes[3], 2);
 760         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
 761
 762         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
 763         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
 764         // with the old ChannelManager.
 765         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
 766         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 767                 if outpoint.to_channel_id() == chan_id_persisted {
 768                         assert!(updated_monitor.0.is_empty());
 769                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
 770                 }
 771         }
 772         // If `persist_both_monitors` is set, get the second monitor here as well
 773         if persist_both_monitors {
 774                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 775                         if outpoint.to_channel_id() == chan_id_not_persisted {
 776                                 assert!(original_monitor.0.is_empty());
 777                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 778                         }
 779                 }
 780         }
 781
 782         // Now restart nodes[3].
 783         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
 784
 785         // On startup the preimage should have been copied into the non-persisted monitor:
 786         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
 787         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
 788
 789         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id());
 790         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id());
 791
 792         // During deserialization, we should have closed one channel and broadcast its latest
 793         // commitment transaction. We should also still have the original PaymentClaimable event we
 794         // never finished processing.
 795         let events = nodes[3].node.get_and_clear_pending_events();
 796         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
 797         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
 798         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
 799         if persist_both_monitors {
 800                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
 801                 check_added_monitors(&nodes[3], 2);
 802         } else {
 803                 check_added_monitors(&nodes[3], 1);
 804         }
 805
 806         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
 807         // ChannelManager prior to handling the original one.
 808         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
 809                 events[if persist_both_monitors { 3 } else { 2 }]
 810         {
 811                 assert_eq!(payment_hash, our_payment_hash);
 812         } else { panic!(); }
 813
 814         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
 815         if !persist_both_monitors {
 816                 // If one of the two channels is still live, reveal the payment preimage over it.
 817
 818                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init {
 819                         features: nodes[2].node.init_features(), networks: None, remote_network_address: None
 820                 }, true).unwrap();
 821                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
 822                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init {
 823                         features: nodes[3].node.init_features(), networks: None, remote_network_address: None
 824                 }, false).unwrap();
 825                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
 826
 827                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
 828                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
 829                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 830
 831                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
 832
 833                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
 834                 // claim should fly.
 835                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
 836                 check_added_monitors!(nodes[3], 1);
 837                 assert_eq!(ds_msgs.len(), 2);
 838                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
 839
 840                 let cs_updates = match ds_msgs[1] {
 841                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
 842                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 843                                 check_added_monitors!(nodes[2], 1);
 844                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 845                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
 846                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
 847                                 cs_updates
 848                         }
 849                         _ => panic!(),
 850                 };
 851
 852                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
 853                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
 854                 expect_payment_sent!(nodes[0], payment_preimage);
 855         }
 856 }
 857
 858 #[test]
 859 fn test_partial_claim_before_restart() {
 860         do_test_partial_claim_before_restart(false);
 861         do_test_partial_claim_before_restart(true);
 862 }
 863
 864 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
 865         if !use_cs_commitment { assert!(!claim_htlc); }
 866         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
 867         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
 868         // it back until the ChannelMonitor decides the fate of the HTLC.
 869         // This was never an issue, but it may be easy to regress here going forward.
 870         let chanmon_cfgs = create_chanmon_cfgs(3);
 871         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 872         let mut intercept_forwards_config = test_default_channel_config();
 873         intercept_forwards_config.accept_intercept_htlcs = true;
 874         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
 875
 876         let persister;
 877         let new_chain_monitor;
 878         let nodes_1_deserialized;
 879
 880         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 881
 882         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
 883         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
 884
 885         let intercept_scid = nodes[1].node.get_intercept_scid();
 886
 887         let (mut route, payment_hash, payment_preimage, payment_secret) =
 888                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 889         if use_intercept {
 890                 route.paths[0].hops[1].short_channel_id = intercept_scid;
 891         }
 892         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 893         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
 894         nodes[0].node.send_payment_with_route(&route, payment_hash,
 895                 RecipientOnionFields::secret_only(payment_secret), payment_id).unwrap();
 896         check_added_monitors!(nodes[0], 1);
 897
 898         let payment_event = SendEvent::from_node(&nodes[0]);
 899         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 900         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 901
 902         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
 903         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
 904         // present when we serialized.
 905         let node_encoded = nodes[1].node.encode();
 906
 907         let mut intercept_id = None;
 908         let mut expected_outbound_amount_msat = None;
 909         if use_intercept {
 910                 let events = nodes[1].node.get_and_clear_pending_events();
 911                 assert_eq!(events.len(), 1);
 912                 match events[0] {
 913                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
 914                                 intercept_id = Some(ev_id);
 915                                 expected_outbound_amount_msat = Some(ev_amt);
 916                         },
 917                         _ => panic!()
 918                 }
 919                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
 920                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
 921         }
 922
 923         expect_pending_htlcs_forwardable!(nodes[1]);
 924
 925         let payment_event = SendEvent::from_node(&nodes[1]);
 926         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 927         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
 928         check_added_monitors!(nodes[2], 1);
 929
 930         if claim_htlc {
 931                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
 932                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
 933         }
 934         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 935
 936         let _ = nodes[2].node.get_and_clear_pending_msg_events();
 937
 938         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
 939         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 940         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
 941
 942         check_added_monitors!(nodes[2], 1);
 943         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
 944         check_closed_broadcast!(nodes[2], true);
 945
 946         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 947         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 948         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 949
 950         // Note that this checks that this is the only event on nodes[1], implying the
 951         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
 952         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
 953
 954         if use_intercept {
 955                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
 956                 // a intercept-doesn't-exist error.
 957                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
 958                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
 959                 assert_eq!(forward_err, APIError::APIMisuseError {
 960                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
 961                 });
 962         }
 963
 964         nodes[1].node.timer_tick_occurred();
 965         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 966         assert_eq!(bs_commitment_tx.len(), 1);
 967         check_added_monitors!(nodes[1], 1);
 968
 969         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
 970         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 971
 972         if use_cs_commitment {
 973                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
 974                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
 975                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
 976                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 977                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 978
 979                 if claim_htlc {
 980                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
 981                 } else {
 982                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1 + 1);
 983                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 984                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
 985                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
 986                 }
 987         } else {
 988                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
 989         }
 990
 991         if !claim_htlc {
 992                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 993         } else {
 994                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
 995         }
 996         check_added_monitors!(nodes[1], 1);
 997
 998         let events = nodes[1].node.get_and_clear_pending_msg_events();
 999         assert_eq!(events.len(), 1);
1000         match &events[0] {
1001                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
1002                         if claim_htlc {
1003                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
1004                         } else {
1005                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1006                         }
1007                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1008                 },
1009                 _ => panic!("Unexpected event"),
1010         }
1011
1012         if claim_htlc {
1013                 expect_payment_sent!(nodes[0], payment_preimage);
1014         } else {
1015                 expect_payment_failed!(nodes[0], payment_hash, false);
1016         }
1017 }
1018
1019 #[test]
1020 fn forwarded_payment_no_manager_persistence() {
1021         do_forwarded_payment_no_manager_persistence(true, true, false);
1022         do_forwarded_payment_no_manager_persistence(true, false, false);
1023         do_forwarded_payment_no_manager_persistence(false, false, false);
1024 }
1025
1026 #[test]
1027 fn intercepted_payment_no_manager_persistence() {
1028         do_forwarded_payment_no_manager_persistence(true, true, true);
1029         do_forwarded_payment_no_manager_persistence(true, false, true);
1030         do_forwarded_payment_no_manager_persistence(false, false, true);
1031 }
1032
1033 #[test]
1034 fn removed_payment_no_manager_persistence() {
1035         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
1036         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
1037         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
1038         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
1039         // were left dangling when a channel was force-closed due to a stale ChannelManager.
1040         let chanmon_cfgs = create_chanmon_cfgs(3);
1041         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
1042         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
1043
1044         let persister;
1045         let new_chain_monitor;
1046         let nodes_1_deserialized;
1047
1048         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1049
1050         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1).2;
1051         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2).2;
1052
1053         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1054
1055         let node_encoded = nodes[1].node.encode();
1056
1057         nodes[2].node.fail_htlc_backwards(&payment_hash);
1058         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1059         check_added_monitors!(nodes[2], 1);
1060         let events = nodes[2].node.get_and_clear_pending_msg_events();
1061         assert_eq!(events.len(), 1);
1062         match &events[0] {
1063                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1064                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1065                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1066                 },
1067                 _ => panic!("Unexpected event"),
1068         }
1069
1070         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1071         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1072         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1073
1074         match nodes[1].node.pop_pending_event().unwrap() {
1075                 Event::ChannelClosed { ref reason, .. } => {
1076                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1077                 },
1078                 _ => panic!("Unexpected event"),
1079         }
1080
1081         nodes[1].node.test_process_background_events();
1082         check_added_monitors(&nodes[1], 1);
1083
1084         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1085         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1086         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1087         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id());
1088         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1089
1090         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1091         check_added_monitors!(nodes[1], 1);
1092         let events = nodes[1].node.get_and_clear_pending_msg_events();
1093         assert_eq!(events.len(), 1);
1094         match &events[0] {
1095                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1096                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1097                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1098                 },
1099                 _ => panic!("Unexpected event"),
1100         }
1101
1102         expect_payment_failed!(nodes[0], payment_hash, false);
1103 }