_ Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test for correct behavior across node restarts.
  11
  12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
  13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
  14 use crate::chain::channelmonitor::ChannelMonitor;
  15 use crate::chain::keysinterface::KeysInterface;
  16 use crate::chain::transaction::OutPoint;
  17 use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, PaymentId};
  18 use crate::ln::msgs;
  19 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
  20 use crate::util::enforcing_trait_impls::EnforcingSigner;
  21 use crate::util::test_utils;
  22 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  23 use crate::util::ser::{Writeable, ReadableArgs};
  24 use crate::util::config::UserConfig;
  25
  26 use bitcoin::hash_types::BlockHash;
  27
  28 use crate::prelude::*;
  29 use core::default::Default;
  30 use crate::sync::Mutex;
  31
  32 use crate::ln::functional_test_utils::*;
  33
  34 #[test]
  35 fn test_funding_peer_disconnect() {
  36         // Test that we can lock in our funding tx while disconnected
  37         let chanmon_cfgs = create_chanmon_cfgs(2);
  38         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  39         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  40         let persister: test_utils::TestPersister;
  41         let new_chain_monitor: test_utils::TestChainMonitor;
  42         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
  43         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  44         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
  45
  46         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  47         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  48
  49         confirm_transaction(&nodes[0], &tx);
  50         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  51         assert!(events_1.is_empty());
  52
  53         reconnect_nodes(&nodes[0], &nodes[1], (false, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  54
  55         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  56         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  57
  58         confirm_transaction(&nodes[1], &tx);
  59         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
  60         assert!(events_2.is_empty());
  61
  62         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
  63         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
  64         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
  65         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
  66
  67         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
  68         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
  69         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
  70         assert_eq!(events_3.len(), 1);
  71         let as_channel_ready = match events_3[0] {
  72                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  73                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
  74                         msg.clone()
  75                 },
  76                 _ => panic!("Unexpected event {:?}", events_3[0]),
  77         };
  78
  79         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
  80         // announcement_signatures as well as channel_update.
  81         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
  82         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
  83         assert_eq!(events_4.len(), 3);
  84         let chan_id;
  85         let bs_channel_ready = match events_4[0] {
  86                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  87                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  88                         chan_id = msg.channel_id;
  89                         msg.clone()
  90                 },
  91                 _ => panic!("Unexpected event {:?}", events_4[0]),
  92         };
  93         let bs_announcement_sigs = match events_4[1] {
  94                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
  95                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  96                         msg.clone()
  97                 },
  98                 _ => panic!("Unexpected event {:?}", events_4[1]),
  99         };
 100         match events_4[2] {
 101                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 102                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 103                 },
 104                 _ => panic!("Unexpected event {:?}", events_4[2]),
 105         }
 106
 107         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
 108         // generates a duplicative private channel_update
 109         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
 110         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
 111         assert_eq!(events_5.len(), 1);
 112         match events_5[0] {
 113                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 114                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 115                 },
 116                 _ => panic!("Unexpected event {:?}", events_5[0]),
 117         };
 118
 119         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
 120         // announcement_signatures.
 121         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
 122         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
 123         assert_eq!(events_6.len(), 1);
 124         let as_announcement_sigs = match events_6[0] {
 125                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 126                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 127                         msg.clone()
 128                 },
 129                 _ => panic!("Unexpected event {:?}", events_6[0]),
 130         };
 131         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
 132         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 133
 134         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
 135         // broadcast the channel announcement globally, as well as re-send its (now-public)
 136         // channel_update.
 137         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
 138         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
 139         assert_eq!(events_7.len(), 1);
 140         let (chan_announcement, as_update) = match events_7[0] {
 141                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 142                         (msg.clone(), update_msg.clone())
 143                 },
 144                 _ => panic!("Unexpected event {:?}", events_7[0]),
 145         };
 146
 147         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
 148         // same channel_announcement.
 149         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
 150         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
 151         assert_eq!(events_8.len(), 1);
 152         let bs_update = match events_8[0] {
 153                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 154                         assert_eq!(*msg, chan_announcement);
 155                         update_msg.clone()
 156                 },
 157                 _ => panic!("Unexpected event {:?}", events_8[0]),
 158         };
 159
 160         // Provide the channel announcement and public updates to the network graph
 161         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
 162         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
 163         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
 164
 165         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
 166         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
 167         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 168
 169         // Check that after deserialization and reconnection we can still generate an identical
 170         // channel_announcement from the cached signatures.
 171         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 172
 173         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 174
 175         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 176
 177         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 178 }
 179
 180 #[test]
 181 fn test_no_txn_manager_serialize_deserialize() {
 182         let chanmon_cfgs = create_chanmon_cfgs(2);
 183         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 184         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 185         let persister: test_utils::TestPersister;
 186         let new_chain_monitor: test_utils::TestChainMonitor;
 187         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 188         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 189
 190         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 191
 192         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 193
 194         let chan_0_monitor_serialized =
 195                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
 196         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 197
 198         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 199         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 200         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 201         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 202
 203         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 204         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 205         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 206         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 207
 208         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 209         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 210         for node in nodes.iter() {
 211                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 212                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 213                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 214         }
 215
 216         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 217 }
 218
 219 #[test]
 220 fn test_manager_serialize_deserialize_events() {
 221         // This test makes sure the events field in ChannelManager survives de/serialization
 222         let chanmon_cfgs = create_chanmon_cfgs(2);
 223         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 224         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 225         let persister: test_utils::TestPersister;
 226         let new_chain_monitor: test_utils::TestChainMonitor;
 227         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 228         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 229
 230         // Start creating a channel, but stop right before broadcasting the funding transaction
 231         let channel_value = 100000;
 232         let push_msat = 10001;
 233         let a_flags = channelmanager::provided_init_features();
 234         let b_flags = channelmanager::provided_init_features();
 235         let node_a = nodes.remove(0);
 236         let node_b = nodes.remove(0);
 237         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
 238         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), a_flags, &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
 239         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), b_flags, &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
 240
 241         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
 242
 243         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
 244         check_added_monitors!(node_a, 0);
 245
 246         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
 247         {
 248                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
 249                 assert_eq!(added_monitors.len(), 1);
 250                 assert_eq!(added_monitors[0].0, funding_output);
 251                 added_monitors.clear();
 252         }
 253
 254         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
 255         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
 256         {
 257                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
 258                 assert_eq!(added_monitors.len(), 1);
 259                 assert_eq!(added_monitors[0].0, funding_output);
 260                 added_monitors.clear();
 261         }
 262         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
 263
 264         nodes.push(node_a);
 265         nodes.push(node_b);
 266
 267         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
 268         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
 269         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 270
 271         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 272
 273         // After deserializing, make sure the funding_transaction is still held by the channel manager
 274         let events_4 = nodes[0].node.get_and_clear_pending_events();
 275         assert_eq!(events_4.len(), 0);
 276         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
 277         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
 278
 279         // Make sure the channel is functioning as though the de/serialization never happened
 280         assert_eq!(nodes[0].node.list_channels().len(), 1);
 281
 282         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 283         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 284         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 285         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 286
 287         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 288         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 289         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 290         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 291
 292         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 293         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 294         for node in nodes.iter() {
 295                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 296                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 297                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 298         }
 299
 300         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 301 }
 302
 303 #[test]
 304 fn test_simple_manager_serialize_deserialize() {
 305         let chanmon_cfgs = create_chanmon_cfgs(2);
 306         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 307         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 308         let persister: test_utils::TestPersister;
 309         let new_chain_monitor: test_utils::TestChainMonitor;
 310         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 311         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 312         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 313
 314         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 315         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 316
 317         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 318
 319         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 320         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 321
 322         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 323
 324         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
 325         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
 326 }
 327
 328 #[test]
 329 fn test_manager_serialize_deserialize_inconsistent_monitor() {
 330         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
 331         let chanmon_cfgs = create_chanmon_cfgs(4);
 332         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 333         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 334         let logger: test_utils::TestLogger;
 335         let fee_estimator: test_utils::TestFeeEstimator;
 336         let persister: test_utils::TestPersister;
 337         let new_chain_monitor: test_utils::TestChainMonitor;
 338         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 339         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 340         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 341         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 342         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 343
 344         let mut node_0_stale_monitors_serialized = Vec::new();
 345         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 346                 let mut writer = test_utils::TestVecWriter(Vec::new());
 347                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
 348                 node_0_stale_monitors_serialized.push(writer.0);
 349         }
 350
 351         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
 352
 353         // Serialize the ChannelManager here, but the monitor we keep up-to-date
 354         let nodes_0_serialized = nodes[0].node.encode();
 355
 356         route_payment(&nodes[0], &[&nodes[3]], 1000000);
 357         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 358         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 359         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 360
 361         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
 362         // nodes[3])
 363         let mut node_0_monitors_serialized = Vec::new();
 364         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 365                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
 366         }
 367
 368         logger = test_utils::TestLogger::new();
 369         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
 370         persister = test_utils::TestPersister::new();
 371         let keys_manager = &chanmon_cfgs[0].keys_manager;
 372         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster.clone(), &logger, &fee_estimator, &persister, keys_manager);
 373         nodes[0].chain_monitor = &new_chain_monitor;
 374
 375
 376         let mut node_0_stale_monitors = Vec::new();
 377         for serialized in node_0_stale_monitors_serialized.iter() {
 378                 let mut read = &serialized[..];
 379                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, keys_manager).unwrap();
 380                 assert!(read.is_empty());
 381                 node_0_stale_monitors.push(monitor);
 382         }
 383
 384         let mut node_0_monitors = Vec::new();
 385         for serialized in node_0_monitors_serialized.iter() {
 386                 let mut read = &serialized[..];
 387                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, keys_manager).unwrap();
 388                 assert!(read.is_empty());
 389                 node_0_monitors.push(monitor);
 390         }
 391
 392         let mut nodes_0_read = &nodes_0_serialized[..];
 393         if let Err(msgs::DecodeError::InvalidValue) =
 394                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 395                 default_config: UserConfig::default(),
 396                 keys_manager,
 397                 fee_estimator: &fee_estimator,
 398                 chain_monitor: nodes[0].chain_monitor,
 399                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 400                 logger: &logger,
 401                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 402         }) { } else {
 403                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
 404         };
 405
 406         let mut nodes_0_read = &nodes_0_serialized[..];
 407         let (_, nodes_0_deserialized_tmp) =
 408                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 409                 default_config: UserConfig::default(),
 410                 keys_manager,
 411                 fee_estimator: &fee_estimator,
 412                 chain_monitor: nodes[0].chain_monitor,
 413                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 414                 logger: &logger,
 415                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 416         }).unwrap();
 417         nodes_0_deserialized = nodes_0_deserialized_tmp;
 418         assert!(nodes_0_read.is_empty());
 419
 420         { // Channel close should result in a commitment tx
 421                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 422                 assert_eq!(txn.len(), 1);
 423                 check_spends!(txn[0], funding_tx);
 424                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
 425         }
 426
 427         for monitor in node_0_monitors.drain(..) {
 428                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
 429                         ChannelMonitorUpdateStatus::Completed);
 430                 check_added_monitors!(nodes[0], 1);
 431         }
 432         nodes[0].node = &nodes_0_deserialized;
 433         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 434
 435         // nodes[1] and nodes[2] have no lost state with nodes[0]...
 436         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 437         reconnect_nodes(&nodes[0], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 438         //... and we can even still claim the payment!
 439         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
 440
 441         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 442         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
 443         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 444         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
 445         let mut found_err = false;
 446         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
 447                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
 448                         match action {
 449                                 &ErrorAction::SendErrorMessage { ref msg } => {
 450                                         assert_eq!(msg.channel_id, channel_id);
 451                                         assert!(!found_err);
 452                                         found_err = true;
 453                                 },
 454                                 _ => panic!("Unexpected event!"),
 455                         }
 456                 }
 457         }
 458         assert!(found_err);
 459 }
 460
 461 fn do_test_data_loss_protect(reconnect_panicing: bool) {
 462         // When we get a data_loss_protect proving we're behind, we immediately panic as the
 463         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
 464         // panic message informs the user they should force-close without broadcasting, which is tested
 465         // if `reconnect_panicing` is not set.
 466         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 467         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
 468         // during signing due to revoked tx
 469         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
 470         let persister;
 471         let new_chain_monitor;
 472         let nodes_0_deserialized;
 473         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 474         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 475         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 476
 477         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 478
 479         // Cache node A state before any channel update
 480         let previous_node_state = nodes[0].node.encode();
 481         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
 482
 483         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 484         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 485
 486         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 487         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 488
 489         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
 490
 491         if reconnect_panicing {
 492                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 493                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 494
 495                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 496
 497                 // Check we close channel detecting A is fallen-behind
 498                 // Check that we sent the warning message when we detected that A has fallen behind,
 499                 // and give the possibility for A to recover from the warning.
 500                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 501                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
 502                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
 503
 504                 {
 505                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
 506                         // The node B should not broadcast the transaction to force close the channel!
 507                         assert!(node_txn.is_empty());
 508                 }
 509
 510                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 511                 // Check A panics upon seeing proof it has fallen behind.
 512                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
 513                 return; // By this point we should have panic'ed!
 514         }
 515
 516         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
 517         check_added_monitors!(nodes[0], 1);
 518         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 519         {
 520                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 521                 assert_eq!(node_txn.len(), 0);
 522         }
 523
 524         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 525                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
 526                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
 527                         match action {
 528                                 &ErrorAction::SendErrorMessage { ref msg } => {
 529                                         assert_eq!(msg.data, "Channel force-closed");
 530                                 },
 531                                 _ => panic!("Unexpected event!"),
 532                         }
 533                 } else {
 534                         panic!("Unexpected event {:?}", msg)
 535                 }
 536         }
 537
 538         // after the warning message sent by B, we should not able to
 539         // use the channel, or reconnect with success to the channel.
 540         assert!(nodes[0].node.list_usable_channels().is_empty());
 541         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 542         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 543         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 544
 545         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
 546         let mut err_msgs_0 = Vec::with_capacity(1);
 547         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 548                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
 549                         match action {
 550                                 &ErrorAction::SendErrorMessage { ref msg } => {
 551                                         assert_eq!(msg.data, "Failed to find corresponding channel");
 552                                         err_msgs_0.push(msg.clone());
 553                                 },
 554                                 _ => panic!("Unexpected event!"),
 555                         }
 556                 } else {
 557                         panic!("Unexpected event!");
 558                 }
 559         }
 560         assert_eq!(err_msgs_0.len(), 1);
 561         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
 562         assert!(nodes[1].node.list_usable_channels().is_empty());
 563         check_added_monitors!(nodes[1], 1);
 564         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: "Failed to find corresponding channel".to_owned() });
 565         check_closed_broadcast!(nodes[1], false);
 566 }
 567
 568 #[test]
 569 #[should_panic]
 570 fn test_data_loss_protect_showing_stale_state_panics() {
 571         do_test_data_loss_protect(true);
 572 }
 573
 574 #[test]
 575 fn test_force_close_without_broadcast() {
 576         do_test_data_loss_protect(false);
 577 }
 578
 579 #[test]
 580 fn test_forwardable_regen() {
 581         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
 582         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
 583         // HTLCs.
 584         // We test it for both payment receipt and payment forwarding.
 585
 586         let chanmon_cfgs = create_chanmon_cfgs(3);
 587         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 588         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 589         let persister: test_utils::TestPersister;
 590         let new_chain_monitor: test_utils::TestChainMonitor;
 591         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 592         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 593         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 594         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 595
 596         // First send a payment to nodes[1]
 597         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 598         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 599         check_added_monitors!(nodes[0], 1);
 600
 601         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 602         assert_eq!(events.len(), 1);
 603         let payment_event = SendEvent::from_event(events.pop().unwrap());
 604         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 605         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 606
 607         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
 608
 609         // Next send a payment which is forwarded by nodes[1]
 610         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
 611         nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 612         check_added_monitors!(nodes[0], 1);
 613
 614         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 615         assert_eq!(events.len(), 1);
 616         let payment_event = SendEvent::from_event(events.pop().unwrap());
 617         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 618         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 619
 620         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
 621         // generated
 622         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 623
 624         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
 625         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 626         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 627
 628         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 629         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 630         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 631
 632         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 633         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
 634         // the commitment state.
 635         reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 636
 637         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 638
 639         expect_pending_htlcs_forwardable!(nodes[1]);
 640         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
 641         check_added_monitors!(nodes[1], 1);
 642
 643         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
 644         assert_eq!(events.len(), 1);
 645         let payment_event = SendEvent::from_event(events.pop().unwrap());
 646         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 647         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
 648         expect_pending_htlcs_forwardable!(nodes[2]);
 649         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
 650
 651         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 652         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
 653 }
 654
 655 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 656         // Test what happens if a node receives an MPP payment, claims it, but crashes before
 657         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
 658         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
 659         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
 660         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
 661         // not have the preimage tied to the still-pending HTLC.
 662         //
 663         // To get to the correct state, on startup we should propagate the preimage to the
 664         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
 665         // receiving the preimage without a state update.
 666         //
 667         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
 668         // definitely claimed.
 669         let chanmon_cfgs = create_chanmon_cfgs(4);
 670         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 671         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 672
 673         let persister: test_utils::TestPersister;
 674         let new_chain_monitor: test_utils::TestChainMonitor;
 675         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestLogger>;
 676
 677         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 678
 679         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 680         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 681         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 682         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 683
 684         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
 685         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
 686         assert_eq!(route.paths.len(), 2);
 687         route.paths.sort_by(|path_a, _| {
 688                 // Sort the path so that the path through nodes[1] comes first
 689                 if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
 690                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
 691         });
 692
 693         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 694         check_added_monitors!(nodes[0], 2);
 695
 696         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
 697         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
 698         assert_eq!(send_events.len(), 2);
 699         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), send_events[0].clone(), true, false, None);
 700         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), send_events[1].clone(), true, false, None);
 701
 702         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
 703         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
 704         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
 705         if !persist_both_monitors {
 706                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 707                         if outpoint.to_channel_id() == chan_id_not_persisted {
 708                                 assert!(original_monitor.0.is_empty());
 709                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 710                         }
 711                 }
 712         }
 713
 714         let original_manager = nodes[3].node.encode();
 715
 716         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
 717
 718         nodes[3].node.claim_funds(payment_preimage);
 719         check_added_monitors!(nodes[3], 2);
 720         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
 721
 722         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
 723         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
 724         // with the old ChannelManager.
 725         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
 726         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 727                 if outpoint.to_channel_id() == chan_id_persisted {
 728                         assert!(updated_monitor.0.is_empty());
 729                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
 730                 }
 731         }
 732         // If `persist_both_monitors` is set, get the second monitor here as well
 733         if persist_both_monitors {
 734                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 735                         if outpoint.to_channel_id() == chan_id_not_persisted {
 736                                 assert!(original_monitor.0.is_empty());
 737                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 738                         }
 739                 }
 740         }
 741
 742         // Now restart nodes[3].
 743         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
 744
 745         // On startup the preimage should have been copied into the non-persisted monitor:
 746         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
 747         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
 748
 749         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id(), false);
 750         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id(), false);
 751
 752         // During deserialization, we should have closed one channel and broadcast its latest
 753         // commitment transaction. We should also still have the original PaymentClaimable event we
 754         // never finished processing.
 755         let events = nodes[3].node.get_and_clear_pending_events();
 756         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
 757         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
 758         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
 759         if persist_both_monitors {
 760                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
 761         }
 762
 763         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
 764         // ChannelManager prior to handling the original one.
 765         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
 766                 events[if persist_both_monitors { 3 } else { 2 }]
 767         {
 768                 assert_eq!(payment_hash, our_payment_hash);
 769         } else { panic!(); }
 770
 771         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
 772         if !persist_both_monitors {
 773                 // If one of the two channels is still live, reveal the payment preimage over it.
 774
 775                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 776                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
 777                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 778                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
 779
 780                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
 781                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
 782                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 783
 784                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
 785
 786                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
 787                 // claim should fly.
 788                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
 789                 check_added_monitors!(nodes[3], 1);
 790                 assert_eq!(ds_msgs.len(), 2);
 791                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
 792
 793                 let cs_updates = match ds_msgs[1] {
 794                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
 795                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 796                                 check_added_monitors!(nodes[2], 1);
 797                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 798                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
 799                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
 800                                 cs_updates
 801                         }
 802                         _ => panic!(),
 803                 };
 804
 805                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
 806                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
 807                 expect_payment_sent!(nodes[0], payment_preimage);
 808         }
 809 }
 810
 811 #[test]
 812 fn test_partial_claim_before_restart() {
 813         do_test_partial_claim_before_restart(false);
 814         do_test_partial_claim_before_restart(true);
 815 }
 816
 817 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool) {
 818         if !use_cs_commitment { assert!(!claim_htlc); }
 819         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
 820         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
 821         // it back until the ChannelMonitor decides the fate of the HTLC.
 822         // This was never an issue, but it may be easy to regress here going forward.
 823         let chanmon_cfgs = create_chanmon_cfgs(3);
 824         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 825         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 826
 827         let persister;
 828         let new_chain_monitor;
 829         let nodes_1_deserialized;
 830
 831         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 832
 833         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 834         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 835
 836         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 837         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 838         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
 839         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 840         check_added_monitors!(nodes[0], 1);
 841
 842         let payment_event = SendEvent::from_node(&nodes[0]);
 843         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 844         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 845
 846         let node_encoded = nodes[1].node.encode();
 847
 848         expect_pending_htlcs_forwardable!(nodes[1]);
 849
 850         let payment_event = SendEvent::from_node(&nodes[1]);
 851         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 852         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
 853         check_added_monitors!(nodes[2], 1);
 854
 855         if claim_htlc {
 856                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
 857                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
 858         }
 859         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 860
 861         let _ = nodes[2].node.get_and_clear_pending_msg_events();
 862
 863         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
 864         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 865         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
 866
 867         check_added_monitors!(nodes[2], 1);
 868         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
 869         check_closed_broadcast!(nodes[2], true);
 870
 871         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 872         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 873         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 874
 875         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
 876
 877         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 878         assert_eq!(bs_commitment_tx.len(), 1);
 879
 880         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
 881         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 882
 883         if use_cs_commitment {
 884                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
 885                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
 886                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
 887                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 888                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 889
 890                 if claim_htlc {
 891                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
 892                 } else {
 893                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
 894                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 895                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
 896                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
 897                 }
 898         } else {
 899                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
 900         }
 901
 902         if !claim_htlc {
 903                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 904         } else {
 905                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
 906         }
 907         check_added_monitors!(nodes[1], 1);
 908
 909         let events = nodes[1].node.get_and_clear_pending_msg_events();
 910         assert_eq!(events.len(), 1);
 911         match &events[0] {
 912                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
 913                         if claim_htlc {
 914                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 915                         } else {
 916                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
 917                         }
 918                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
 919                 },
 920                 _ => panic!("Unexpected event"),
 921         }
 922
 923         if claim_htlc {
 924                 expect_payment_sent!(nodes[0], payment_preimage);
 925         } else {
 926                 expect_payment_failed!(nodes[0], payment_hash, false);
 927         }
 928 }
 929
 930 #[test]
 931 fn forwarded_payment_no_manager_persistence() {
 932         do_forwarded_payment_no_manager_persistence(true, true);
 933         do_forwarded_payment_no_manager_persistence(true, false);
 934         do_forwarded_payment_no_manager_persistence(false, false);
 935 }
 936
 937 #[test]
 938 fn removed_payment_no_manager_persistence() {
 939         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
 940         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
 941         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
 942         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
 943         // were left dangling when a channel was force-closed due to a stale ChannelManager.
 944         let chanmon_cfgs = create_chanmon_cfgs(3);
 945         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 946         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 947
 948         let persister;
 949         let new_chain_monitor;
 950         let nodes_1_deserialized;
 951
 952         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 953
 954         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 955         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 956
 957         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
 958
 959         let node_encoded = nodes[1].node.encode();
 960
 961         nodes[2].node.fail_htlc_backwards(&payment_hash);
 962         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
 963         check_added_monitors!(nodes[2], 1);
 964         let events = nodes[2].node.get_and_clear_pending_msg_events();
 965         assert_eq!(events.len(), 1);
 966         match &events[0] {
 967                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
 968                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
 969                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
 970                 },
 971                 _ => panic!("Unexpected event"),
 972         }
 973
 974         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 975         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 976         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 977
 978         match nodes[1].node.pop_pending_event().unwrap() {
 979                 Event::ChannelClosed { ref reason, .. } => {
 980                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
 981                 },
 982                 _ => panic!("Unexpected event"),
 983         }
 984
 985         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
 986         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
 987         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
 988         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
 989         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 990
 991         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 992         check_added_monitors!(nodes[1], 1);
 993         let events = nodes[1].node.get_and_clear_pending_msg_events();
 994         assert_eq!(events.len(), 1);
 995         match &events[0] {
 996                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
 997                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
 998                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
 999                 },
1000                 _ => panic!("Unexpected event"),
1001         }
1002
1003         expect_payment_failed!(nodes[0], payment_hash, false);
1004 }