_ Git - rust-lightning/blob - lightning/src/ln/reload_tests.rs

   1 // This file is Copyright its original authors, visible in version control
   2 // history.
   3 //
   4 // This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
   5 // or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   6 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
   7 // You may not use this file except in accordance with one or both of these
   8 // licenses.
   9
  10 //! Functional tests which test for correct behavior across node restarts.
  11
  12 use crate::chain::{ChannelMonitorUpdateStatus, Watch};
  13 use crate::chain::chaininterface::LowerBoundedFeeEstimator;
  14 use crate::chain::channelmonitor::ChannelMonitor;
  15 use crate::chain::keysinterface::{EntropySource, KeysInterface};
  16 use crate::chain::transaction::OutPoint;
  17 use crate::ln::channelmanager::{self, ChannelManager, ChannelManagerReadArgs, PaymentId};
  18 use crate::ln::msgs;
  19 use crate::ln::msgs::{ChannelMessageHandler, RoutingMessageHandler, ErrorAction};
  20 use crate::util::enforcing_trait_impls::EnforcingSigner;
  21 use crate::util::test_utils;
  22 use crate::util::errors::APIError;
  23 use crate::util::events::{ClosureReason, Event, HTLCDestination, MessageSendEvent, MessageSendEventsProvider};
  24 use crate::util::ser::{Writeable, ReadableArgs};
  25 use crate::util::config::UserConfig;
  26
  27 use bitcoin::hash_types::BlockHash;
  28
  29 use crate::prelude::*;
  30 use core::default::Default;
  31 use crate::sync::Mutex;
  32
  33 use crate::ln::functional_test_utils::*;
  34
  35 #[test]
  36 fn test_funding_peer_disconnect() {
  37         // Test that we can lock in our funding tx while disconnected
  38         let chanmon_cfgs = create_chanmon_cfgs(2);
  39         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
  40         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
  41         let persister: test_utils::TestPersister;
  42         let new_chain_monitor: test_utils::TestChainMonitor;
  43         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
  44         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
  45         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
  46
  47         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  48         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  49
  50         confirm_transaction(&nodes[0], &tx);
  51         let events_1 = nodes[0].node.get_and_clear_pending_msg_events();
  52         assert!(events_1.is_empty());
  53
  54         reconnect_nodes(&nodes[0], &nodes[1], (false, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
  55
  56         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
  57         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
  58
  59         confirm_transaction(&nodes[1], &tx);
  60         let events_2 = nodes[1].node.get_and_clear_pending_msg_events();
  61         assert!(events_2.is_empty());
  62
  63         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
  64         let as_reestablish = get_chan_reestablish_msgs!(nodes[0], nodes[1]).pop().unwrap();
  65         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
  66         let bs_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]).pop().unwrap();
  67
  68         // nodes[0] hasn't yet received a channel_ready, so it only sends that on reconnect.
  69         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &bs_reestablish);
  70         let events_3 = nodes[0].node.get_and_clear_pending_msg_events();
  71         assert_eq!(events_3.len(), 1);
  72         let as_channel_ready = match events_3[0] {
  73                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  74                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
  75                         msg.clone()
  76                 },
  77                 _ => panic!("Unexpected event {:?}", events_3[0]),
  78         };
  79
  80         // nodes[1] received nodes[0]'s channel_ready on the first reconnect above, so it should send
  81         // announcement_signatures as well as channel_update.
  82         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &as_reestablish);
  83         let events_4 = nodes[1].node.get_and_clear_pending_msg_events();
  84         assert_eq!(events_4.len(), 3);
  85         let chan_id;
  86         let bs_channel_ready = match events_4[0] {
  87                 MessageSendEvent::SendChannelReady { ref node_id, ref msg } => {
  88                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  89                         chan_id = msg.channel_id;
  90                         msg.clone()
  91                 },
  92                 _ => panic!("Unexpected event {:?}", events_4[0]),
  93         };
  94         let bs_announcement_sigs = match events_4[1] {
  95                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
  96                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
  97                         msg.clone()
  98                 },
  99                 _ => panic!("Unexpected event {:?}", events_4[1]),
 100         };
 101         match events_4[2] {
 102                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 103                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 104                 },
 105                 _ => panic!("Unexpected event {:?}", events_4[2]),
 106         }
 107
 108         // Re-deliver nodes[0]'s channel_ready, which nodes[1] can safely ignore. It currently
 109         // generates a duplicative private channel_update
 110         nodes[1].node.handle_channel_ready(&nodes[0].node.get_our_node_id(), &as_channel_ready);
 111         let events_5 = nodes[1].node.get_and_clear_pending_msg_events();
 112         assert_eq!(events_5.len(), 1);
 113         match events_5[0] {
 114                 MessageSendEvent::SendChannelUpdate { ref node_id, msg: _ } => {
 115                         assert_eq!(*node_id, nodes[0].node.get_our_node_id());
 116                 },
 117                 _ => panic!("Unexpected event {:?}", events_5[0]),
 118         };
 119
 120         // When we deliver nodes[1]'s channel_ready, however, nodes[0] will generate its
 121         // announcement_signatures.
 122         nodes[0].node.handle_channel_ready(&nodes[1].node.get_our_node_id(), &bs_channel_ready);
 123         let events_6 = nodes[0].node.get_and_clear_pending_msg_events();
 124         assert_eq!(events_6.len(), 1);
 125         let as_announcement_sigs = match events_6[0] {
 126                 MessageSendEvent::SendAnnouncementSignatures { ref node_id, ref msg } => {
 127                         assert_eq!(*node_id, nodes[1].node.get_our_node_id());
 128                         msg.clone()
 129                 },
 130                 _ => panic!("Unexpected event {:?}", events_6[0]),
 131         };
 132         expect_channel_ready_event(&nodes[0], &nodes[1].node.get_our_node_id());
 133         expect_channel_ready_event(&nodes[1], &nodes[0].node.get_our_node_id());
 134
 135         // When we deliver nodes[1]'s announcement_signatures to nodes[0], nodes[0] should immediately
 136         // broadcast the channel announcement globally, as well as re-send its (now-public)
 137         // channel_update.
 138         nodes[0].node.handle_announcement_signatures(&nodes[1].node.get_our_node_id(), &bs_announcement_sigs);
 139         let events_7 = nodes[0].node.get_and_clear_pending_msg_events();
 140         assert_eq!(events_7.len(), 1);
 141         let (chan_announcement, as_update) = match events_7[0] {
 142                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 143                         (msg.clone(), update_msg.clone())
 144                 },
 145                 _ => panic!("Unexpected event {:?}", events_7[0]),
 146         };
 147
 148         // Finally, deliver nodes[0]'s announcement_signatures to nodes[1] and make sure it creates the
 149         // same channel_announcement.
 150         nodes[1].node.handle_announcement_signatures(&nodes[0].node.get_our_node_id(), &as_announcement_sigs);
 151         let events_8 = nodes[1].node.get_and_clear_pending_msg_events();
 152         assert_eq!(events_8.len(), 1);
 153         let bs_update = match events_8[0] {
 154                 MessageSendEvent::BroadcastChannelAnnouncement { ref msg, ref update_msg } => {
 155                         assert_eq!(*msg, chan_announcement);
 156                         update_msg.clone()
 157                 },
 158                 _ => panic!("Unexpected event {:?}", events_8[0]),
 159         };
 160
 161         // Provide the channel announcement and public updates to the network graph
 162         nodes[0].gossip_sync.handle_channel_announcement(&chan_announcement).unwrap();
 163         nodes[0].gossip_sync.handle_channel_update(&bs_update).unwrap();
 164         nodes[0].gossip_sync.handle_channel_update(&as_update).unwrap();
 165
 166         let (route, _, _, _) = get_route_and_payment_hash!(nodes[0], nodes[1], 1000000);
 167         let payment_preimage = send_along_route(&nodes[0], route, &[&nodes[1]], 1000000).0;
 168         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 169
 170         // Check that after deserialization and reconnection we can still generate an identical
 171         // channel_announcement from the cached signatures.
 172         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 173
 174         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 175
 176         reload_node!(nodes[0], &nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 177
 178         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 179 }
 180
 181 #[test]
 182 fn test_no_txn_manager_serialize_deserialize() {
 183         let chanmon_cfgs = create_chanmon_cfgs(2);
 184         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 185         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 186         let persister: test_utils::TestPersister;
 187         let new_chain_monitor: test_utils::TestChainMonitor;
 188         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 189         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 190
 191         let tx = create_chan_between_nodes_with_value_init(&nodes[0], &nodes[1], 100000, 10001, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 192
 193         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 194
 195         let chan_0_monitor_serialized =
 196                 get_monitor!(nodes[0], OutPoint { txid: tx.txid(), index: 0 }.to_channel_id()).encode();
 197         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 198
 199         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 200         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 201         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 202         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 203
 204         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 205         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 206         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 207         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 208
 209         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 210         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 211         for node in nodes.iter() {
 212                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 213                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 214                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 215         }
 216
 217         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 218 }
 219
 220 #[test]
 221 fn test_manager_serialize_deserialize_events() {
 222         // This test makes sure the events field in ChannelManager survives de/serialization
 223         let chanmon_cfgs = create_chanmon_cfgs(2);
 224         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 225         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 226         let persister: test_utils::TestPersister;
 227         let new_chain_monitor: test_utils::TestChainMonitor;
 228         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 229         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 230
 231         // Start creating a channel, but stop right before broadcasting the funding transaction
 232         let channel_value = 100000;
 233         let push_msat = 10001;
 234         let a_flags = channelmanager::provided_init_features();
 235         let b_flags = channelmanager::provided_init_features();
 236         let node_a = nodes.remove(0);
 237         let node_b = nodes.remove(0);
 238         node_a.node.create_channel(node_b.node.get_our_node_id(), channel_value, push_msat, 42, None).unwrap();
 239         node_b.node.handle_open_channel(&node_a.node.get_our_node_id(), a_flags, &get_event_msg!(node_a, MessageSendEvent::SendOpenChannel, node_b.node.get_our_node_id()));
 240         node_a.node.handle_accept_channel(&node_b.node.get_our_node_id(), b_flags, &get_event_msg!(node_b, MessageSendEvent::SendAcceptChannel, node_a.node.get_our_node_id()));
 241
 242         let (temporary_channel_id, tx, funding_output) = create_funding_transaction(&node_a, &node_b.node.get_our_node_id(), channel_value, 42);
 243
 244         node_a.node.funding_transaction_generated(&temporary_channel_id, &node_b.node.get_our_node_id(), tx.clone()).unwrap();
 245         check_added_monitors!(node_a, 0);
 246
 247         node_b.node.handle_funding_created(&node_a.node.get_our_node_id(), &get_event_msg!(node_a, MessageSendEvent::SendFundingCreated, node_b.node.get_our_node_id()));
 248         {
 249                 let mut added_monitors = node_b.chain_monitor.added_monitors.lock().unwrap();
 250                 assert_eq!(added_monitors.len(), 1);
 251                 assert_eq!(added_monitors[0].0, funding_output);
 252                 added_monitors.clear();
 253         }
 254
 255         let bs_funding_signed = get_event_msg!(node_b, MessageSendEvent::SendFundingSigned, node_a.node.get_our_node_id());
 256         node_a.node.handle_funding_signed(&node_b.node.get_our_node_id(), &bs_funding_signed);
 257         {
 258                 let mut added_monitors = node_a.chain_monitor.added_monitors.lock().unwrap();
 259                 assert_eq!(added_monitors.len(), 1);
 260                 assert_eq!(added_monitors[0].0, funding_output);
 261                 added_monitors.clear();
 262         }
 263         // Normally, this is where node_a would broadcast the funding transaction, but the test de/serializes first instead
 264
 265         nodes.push(node_a);
 266         nodes.push(node_b);
 267
 268         // Start the de/seriailization process mid-channel creation to check that the channel manager will hold onto events that are serialized
 269         let chan_0_monitor_serialized = get_monitor!(nodes[0], bs_funding_signed.channel_id).encode();
 270         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 271
 272         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 273
 274         // After deserializing, make sure the funding_transaction is still held by the channel manager
 275         let events_4 = nodes[0].node.get_and_clear_pending_events();
 276         assert_eq!(events_4.len(), 0);
 277         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap().len(), 1);
 278         assert_eq!(nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap()[0].txid(), funding_output.txid);
 279
 280         // Make sure the channel is functioning as though the de/serialization never happened
 281         assert_eq!(nodes[0].node.list_channels().len(), 1);
 282
 283         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 284         let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 285         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 286         let reestablish_2 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 287
 288         nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 289         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 290         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_2[0]);
 291         assert!(nodes[0].node.get_and_clear_pending_msg_events().is_empty());
 292
 293         let (channel_ready, _) = create_chan_between_nodes_with_value_confirm(&nodes[0], &nodes[1], &tx);
 294         let (announcement, as_update, bs_update) = create_chan_between_nodes_with_value_b(&nodes[0], &nodes[1], &channel_ready);
 295         for node in nodes.iter() {
 296                 assert!(node.gossip_sync.handle_channel_announcement(&announcement).unwrap());
 297                 node.gossip_sync.handle_channel_update(&as_update).unwrap();
 298                 node.gossip_sync.handle_channel_update(&bs_update).unwrap();
 299         }
 300
 301         send_payment(&nodes[0], &[&nodes[1]], 1000000);
 302 }
 303
 304 #[test]
 305 fn test_simple_manager_serialize_deserialize() {
 306         let chanmon_cfgs = create_chanmon_cfgs(2);
 307         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 308         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 309         let persister: test_utils::TestPersister;
 310         let new_chain_monitor: test_utils::TestChainMonitor;
 311         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 312         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 313         let chan_id = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 314
 315         let (our_payment_preimage, _, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 316         let (_, our_payment_hash, _) = route_payment(&nodes[0], &[&nodes[1]], 1000000);
 317
 318         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 319
 320         let chan_0_monitor_serialized = get_monitor!(nodes[0], chan_id).encode();
 321         reload_node!(nodes[0], nodes[0].node.encode(), &[&chan_0_monitor_serialized], persister, new_chain_monitor, nodes_0_deserialized);
 322
 323         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 324
 325         fail_payment(&nodes[0], &[&nodes[1]], our_payment_hash);
 326         claim_payment(&nodes[0], &[&nodes[1]], our_payment_preimage);
 327 }
 328
 329 #[test]
 330 fn test_manager_serialize_deserialize_inconsistent_monitor() {
 331         // Test deserializing a ChannelManager with an out-of-date ChannelMonitor
 332         let chanmon_cfgs = create_chanmon_cfgs(4);
 333         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 334         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 335         let logger: test_utils::TestLogger;
 336         let fee_estimator: test_utils::TestFeeEstimator;
 337         let persister: test_utils::TestPersister;
 338         let new_chain_monitor: test_utils::TestChainMonitor;
 339         let nodes_0_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 340         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 341         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 342         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 2, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 343         let (_, _, channel_id, funding_tx) = create_announced_chan_between_nodes(&nodes, 0, 3, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 344
 345         let mut node_0_stale_monitors_serialized = Vec::new();
 346         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 347                 let mut writer = test_utils::TestVecWriter(Vec::new());
 348                 get_monitor!(nodes[0], chan_id_iter).write(&mut writer).unwrap();
 349                 node_0_stale_monitors_serialized.push(writer.0);
 350         }
 351
 352         let (our_payment_preimage, _, _) = route_payment(&nodes[2], &[&nodes[0], &nodes[1]], 1000000);
 353
 354         // Serialize the ChannelManager here, but the monitor we keep up-to-date
 355         let nodes_0_serialized = nodes[0].node.encode();
 356
 357         route_payment(&nodes[0], &[&nodes[3]], 1000000);
 358         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 359         nodes[2].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 360         nodes[3].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 361
 362         // Now the ChannelMonitor (which is now out-of-sync with ChannelManager for channel w/
 363         // nodes[3])
 364         let mut node_0_monitors_serialized = Vec::new();
 365         for chan_id_iter in &[chan_id_1, chan_id_2, channel_id] {
 366                 node_0_monitors_serialized.push(get_monitor!(nodes[0], chan_id_iter).encode());
 367         }
 368
 369         logger = test_utils::TestLogger::new();
 370         fee_estimator = test_utils::TestFeeEstimator { sat_per_kw: Mutex::new(253) };
 371         persister = test_utils::TestPersister::new();
 372         let keys_manager = &chanmon_cfgs[0].keys_manager;
 373         new_chain_monitor = test_utils::TestChainMonitor::new(Some(nodes[0].chain_source), nodes[0].tx_broadcaster.clone(), &logger, &fee_estimator, &persister, keys_manager);
 374         nodes[0].chain_monitor = &new_chain_monitor;
 375
 376
 377         let mut node_0_stale_monitors = Vec::new();
 378         for serialized in node_0_stale_monitors_serialized.iter() {
 379                 let mut read = &serialized[..];
 380                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, keys_manager).unwrap();
 381                 assert!(read.is_empty());
 382                 node_0_stale_monitors.push(monitor);
 383         }
 384
 385         let mut node_0_monitors = Vec::new();
 386         for serialized in node_0_monitors_serialized.iter() {
 387                 let mut read = &serialized[..];
 388                 let (_, monitor) = <(BlockHash, ChannelMonitor<EnforcingSigner>)>::read(&mut read, keys_manager).unwrap();
 389                 assert!(read.is_empty());
 390                 node_0_monitors.push(monitor);
 391         }
 392
 393         let mut nodes_0_read = &nodes_0_serialized[..];
 394         if let Err(msgs::DecodeError::InvalidValue) =
 395                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 396                 default_config: UserConfig::default(),
 397                 keys_manager,
 398                 fee_estimator: &fee_estimator,
 399                 router: &nodes[0].router,
 400                 chain_monitor: nodes[0].chain_monitor,
 401                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 402                 logger: &logger,
 403                 channel_monitors: node_0_stale_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 404         }) { } else {
 405                 panic!("If the monitor(s) are stale, this indicates a bug and we should get an Err return");
 406         };
 407
 408         let mut nodes_0_read = &nodes_0_serialized[..];
 409         let (_, nodes_0_deserialized_tmp) =
 410                 <(BlockHash, ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>)>::read(&mut nodes_0_read, ChannelManagerReadArgs {
 411                 default_config: UserConfig::default(),
 412                 keys_manager,
 413                 fee_estimator: &fee_estimator,
 414                 router: nodes[0].router,
 415                 chain_monitor: nodes[0].chain_monitor,
 416                 tx_broadcaster: nodes[0].tx_broadcaster.clone(),
 417                 logger: &logger,
 418                 channel_monitors: node_0_monitors.iter_mut().map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect(),
 419         }).unwrap();
 420         nodes_0_deserialized = nodes_0_deserialized_tmp;
 421         assert!(nodes_0_read.is_empty());
 422
 423         { // Channel close should result in a commitment tx
 424                 let txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 425                 assert_eq!(txn.len(), 1);
 426                 check_spends!(txn[0], funding_tx);
 427                 assert_eq!(txn[0].input[0].previous_output.txid, funding_tx.txid());
 428         }
 429
 430         for monitor in node_0_monitors.drain(..) {
 431                 assert_eq!(nodes[0].chain_monitor.watch_channel(monitor.get_funding_txo().0, monitor),
 432                         ChannelMonitorUpdateStatus::Completed);
 433                 check_added_monitors!(nodes[0], 1);
 434         }
 435         nodes[0].node = &nodes_0_deserialized;
 436         check_closed_event!(nodes[0], 1, ClosureReason::OutdatedChannelManager);
 437
 438         // nodes[1] and nodes[2] have no lost state with nodes[0]...
 439         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 440         reconnect_nodes(&nodes[0], &nodes[2], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 441         //... and we can even still claim the payment!
 442         claim_payment(&nodes[2], &[&nodes[0], &nodes[1]], our_payment_preimage);
 443
 444         nodes[3].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 445         let reestablish = get_chan_reestablish_msgs!(nodes[3], nodes[0]).pop().unwrap();
 446         nodes[0].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 447         nodes[0].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish);
 448         let mut found_err = false;
 449         for msg_event in nodes[0].node.get_and_clear_pending_msg_events() {
 450                 if let MessageSendEvent::HandleError { ref action, .. } = msg_event {
 451                         match action {
 452                                 &ErrorAction::SendErrorMessage { ref msg } => {
 453                                         assert_eq!(msg.channel_id, channel_id);
 454                                         assert!(!found_err);
 455                                         found_err = true;
 456                                 },
 457                                 _ => panic!("Unexpected event!"),
 458                         }
 459                 }
 460         }
 461         assert!(found_err);
 462 }
 463
 464 fn do_test_data_loss_protect(reconnect_panicing: bool) {
 465         // When we get a data_loss_protect proving we're behind, we immediately panic as the
 466         // chain::Watch API requirements have been violated (e.g. the user restored from a backup). The
 467         // panic message informs the user they should force-close without broadcasting, which is tested
 468         // if `reconnect_panicing` is not set.
 469         let mut chanmon_cfgs = create_chanmon_cfgs(2);
 470         // We broadcast during Drop because chanmon is out of sync with chanmgr, which would cause a panic
 471         // during signing due to revoked tx
 472         chanmon_cfgs[0].keys_manager.disable_revocation_policy_check = true;
 473         let persister;
 474         let new_chain_monitor;
 475         let nodes_0_deserialized;
 476         let node_cfgs = create_node_cfgs(2, &chanmon_cfgs);
 477         let node_chanmgrs = create_node_chanmgrs(2, &node_cfgs, &[None, None]);
 478         let mut nodes = create_network(2, &node_cfgs, &node_chanmgrs);
 479
 480         let chan = create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 1000000, 1000000, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 481
 482         // Cache node A state before any channel update
 483         let previous_node_state = nodes[0].node.encode();
 484         let previous_chain_monitor_state = get_monitor!(nodes[0], chan.2).encode();
 485
 486         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 487         send_payment(&nodes[0], &vec!(&nodes[1])[..], 8000000);
 488
 489         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 490         nodes[1].node.peer_disconnected(&nodes[0].node.get_our_node_id(), false);
 491
 492         reload_node!(nodes[0], previous_node_state, &[&previous_chain_monitor_state], persister, new_chain_monitor, nodes_0_deserialized);
 493
 494         if reconnect_panicing {
 495                 nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 496                 nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 497
 498                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[0], nodes[1]);
 499
 500                 // Check we close channel detecting A is fallen-behind
 501                 // Check that we sent the warning message when we detected that A has fallen behind,
 502                 // and give the possibility for A to recover from the warning.
 503                 nodes[1].node.handle_channel_reestablish(&nodes[0].node.get_our_node_id(), &reestablish_1[0]);
 504                 let warn_msg = "Peer attempted to reestablish channel with a very old local commitment transaction".to_owned();
 505                 assert!(check_warn_msg!(nodes[1], nodes[0].node.get_our_node_id(), chan.2).contains(&warn_msg));
 506
 507                 {
 508                         let mut node_txn = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().clone();
 509                         // The node B should not broadcast the transaction to force close the channel!
 510                         assert!(node_txn.is_empty());
 511                 }
 512
 513                 let reestablish_0 = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 514                 // Check A panics upon seeing proof it has fallen behind.
 515                 nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &reestablish_0[0]);
 516                 return; // By this point we should have panic'ed!
 517         }
 518
 519         nodes[0].node.force_close_without_broadcasting_txn(&chan.2, &nodes[1].node.get_our_node_id()).unwrap();
 520         check_added_monitors!(nodes[0], 1);
 521         check_closed_event!(nodes[0], 1, ClosureReason::HolderForceClosed);
 522         {
 523                 let node_txn = nodes[0].tx_broadcaster.txn_broadcasted.lock().unwrap();
 524                 assert_eq!(node_txn.len(), 0);
 525         }
 526
 527         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 528                 if let MessageSendEvent::BroadcastChannelUpdate { .. } = msg {
 529                 } else if let MessageSendEvent::HandleError { ref action, .. } = msg {
 530                         match action {
 531                                 &ErrorAction::SendErrorMessage { ref msg } => {
 532                                         assert_eq!(msg.data, "Channel force-closed");
 533                                 },
 534                                 _ => panic!("Unexpected event!"),
 535                         }
 536                 } else {
 537                         panic!("Unexpected event {:?}", msg)
 538                 }
 539         }
 540
 541         // after the warning message sent by B, we should not able to
 542         // use the channel, or reconnect with success to the channel.
 543         assert!(nodes[0].node.list_usable_channels().is_empty());
 544         nodes[0].node.peer_connected(&nodes[1].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 545         nodes[1].node.peer_connected(&nodes[0].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 546         let retry_reestablish = get_chan_reestablish_msgs!(nodes[1], nodes[0]);
 547
 548         nodes[0].node.handle_channel_reestablish(&nodes[1].node.get_our_node_id(), &retry_reestablish[0]);
 549         let mut err_msgs_0 = Vec::with_capacity(1);
 550         for msg in nodes[0].node.get_and_clear_pending_msg_events() {
 551                 if let MessageSendEvent::HandleError { ref action, .. } = msg {
 552                         match action {
 553                                 &ErrorAction::SendErrorMessage { ref msg } => {
 554                                         assert_eq!(msg.data, format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()));
 555                                         err_msgs_0.push(msg.clone());
 556                                 },
 557                                 _ => panic!("Unexpected event!"),
 558                         }
 559                 } else {
 560                         panic!("Unexpected event!");
 561                 }
 562         }
 563         assert_eq!(err_msgs_0.len(), 1);
 564         nodes[1].node.handle_error(&nodes[0].node.get_our_node_id(), &err_msgs_0[0]);
 565         assert!(nodes[1].node.list_usable_channels().is_empty());
 566         check_added_monitors!(nodes[1], 1);
 567         check_closed_event!(nodes[1], 1, ClosureReason::CounterpartyForceClosed { peer_msg: format!("Got a message for a channel from the wrong node! No such channel for the passed counterparty_node_id {}", &nodes[1].node.get_our_node_id()) });
 568         check_closed_broadcast!(nodes[1], false);
 569 }
 570
 571 #[test]
 572 #[should_panic]
 573 fn test_data_loss_protect_showing_stale_state_panics() {
 574         do_test_data_loss_protect(true);
 575 }
 576
 577 #[test]
 578 fn test_force_close_without_broadcast() {
 579         do_test_data_loss_protect(false);
 580 }
 581
 582 #[test]
 583 fn test_forwardable_regen() {
 584         // Tests that if we reload a ChannelManager while forwards are pending we will regenerate the
 585         // PendingHTLCsForwardable event automatically, ensuring we don't forget to forward/receive
 586         // HTLCs.
 587         // We test it for both payment receipt and payment forwarding.
 588
 589         let chanmon_cfgs = create_chanmon_cfgs(3);
 590         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 591         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 592         let persister: test_utils::TestPersister;
 593         let new_chain_monitor: test_utils::TestChainMonitor;
 594         let nodes_1_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 595         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 596         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 597         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 598
 599         // First send a payment to nodes[1]
 600         let (route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[1], 100_000);
 601         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 602         check_added_monitors!(nodes[0], 1);
 603
 604         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 605         assert_eq!(events.len(), 1);
 606         let payment_event = SendEvent::from_event(events.pop().unwrap());
 607         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 608         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 609
 610         expect_pending_htlcs_forwardable_ignore!(nodes[1]);
 611
 612         // Next send a payment which is forwarded by nodes[1]
 613         let (route_2, payment_hash_2, payment_preimage_2, payment_secret_2) = get_route_and_payment_hash!(nodes[0], nodes[2], 200_000);
 614         nodes[0].node.send_payment(&route_2, payment_hash_2, &Some(payment_secret_2), PaymentId(payment_hash_2.0)).unwrap();
 615         check_added_monitors!(nodes[0], 1);
 616
 617         let mut events = nodes[0].node.get_and_clear_pending_msg_events();
 618         assert_eq!(events.len(), 1);
 619         let payment_event = SendEvent::from_event(events.pop().unwrap());
 620         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 621         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 622
 623         // There is already a PendingHTLCsForwardable event "pending" so another one will not be
 624         // generated
 625         assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 626
 627         // Now restart nodes[1] and make sure it regenerates a single PendingHTLCsForwardable
 628         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 629         nodes[2].node.peer_disconnected(&nodes[1].node.get_our_node_id(), false);
 630
 631         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 632         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 633         reload_node!(nodes[1], nodes[1].node.encode(), &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 634
 635         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 636         // Note that nodes[1] and nodes[2] resend their channel_ready here since they haven't updated
 637         // the commitment state.
 638         reconnect_nodes(&nodes[1], &nodes[2], (true, true), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 639
 640         assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 641
 642         expect_pending_htlcs_forwardable!(nodes[1]);
 643         expect_payment_claimable!(nodes[1], payment_hash, payment_secret, 100_000);
 644         check_added_monitors!(nodes[1], 1);
 645
 646         let mut events = nodes[1].node.get_and_clear_pending_msg_events();
 647         assert_eq!(events.len(), 1);
 648         let payment_event = SendEvent::from_event(events.pop().unwrap());
 649         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 650         commitment_signed_dance!(nodes[2], nodes[1], payment_event.commitment_msg, false);
 651         expect_pending_htlcs_forwardable!(nodes[2]);
 652         expect_payment_claimable!(nodes[2], payment_hash_2, payment_secret_2, 200_000);
 653
 654         claim_payment(&nodes[0], &[&nodes[1]], payment_preimage);
 655         claim_payment(&nodes[0], &[&nodes[1], &nodes[2]], payment_preimage_2);
 656 }
 657
 658 fn do_test_partial_claim_before_restart(persist_both_monitors: bool) {
 659         // Test what happens if a node receives an MPP payment, claims it, but crashes before
 660         // persisting the ChannelManager. If `persist_both_monitors` is false, also crash after only
 661         // updating one of the two channels' ChannelMonitors. As a result, on startup, we'll (a) still
 662         // have the PaymentClaimable event, (b) have one (or two) channel(s) that goes on chain with the
 663         // HTLC preimage in them, and (c) optionally have one channel that is live off-chain but does
 664         // not have the preimage tied to the still-pending HTLC.
 665         //
 666         // To get to the correct state, on startup we should propagate the preimage to the
 667         // still-off-chain channel, claiming the HTLC as soon as the peer connects, with the monitor
 668         // receiving the preimage without a state update.
 669         //
 670         // Further, we should generate a `PaymentClaimed` event to inform the user that the payment was
 671         // definitely claimed.
 672         let chanmon_cfgs = create_chanmon_cfgs(4);
 673         let node_cfgs = create_node_cfgs(4, &chanmon_cfgs);
 674         let node_chanmgrs = create_node_chanmgrs(4, &node_cfgs, &[None, None, None, None]);
 675
 676         let persister: test_utils::TestPersister;
 677         let new_chain_monitor: test_utils::TestChainMonitor;
 678         let nodes_3_deserialized: ChannelManager<&test_utils::TestChainMonitor, &test_utils::TestBroadcaster, &test_utils::TestKeysInterface, &test_utils::TestFeeEstimator, &test_utils::TestRouter, &test_utils::TestLogger>;
 679
 680         let mut nodes = create_network(4, &node_cfgs, &node_chanmgrs);
 681
 682         create_announced_chan_between_nodes_with_value(&nodes, 0, 1, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 683         create_announced_chan_between_nodes_with_value(&nodes, 0, 2, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features());
 684         let chan_id_persisted = create_announced_chan_between_nodes_with_value(&nodes, 1, 3, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 685         let chan_id_not_persisted = create_announced_chan_between_nodes_with_value(&nodes, 2, 3, 100_000, 0, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 686
 687         // Create an MPP route for 15k sats, more than the default htlc-max of 10%
 688         let (mut route, payment_hash, payment_preimage, payment_secret) = get_route_and_payment_hash!(nodes[0], nodes[3], 15_000_000);
 689         assert_eq!(route.paths.len(), 2);
 690         route.paths.sort_by(|path_a, _| {
 691                 // Sort the path so that the path through nodes[1] comes first
 692                 if path_a[0].pubkey == nodes[1].node.get_our_node_id() {
 693                         core::cmp::Ordering::Less } else { core::cmp::Ordering::Greater }
 694         });
 695
 696         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), PaymentId(payment_hash.0)).unwrap();
 697         check_added_monitors!(nodes[0], 2);
 698
 699         // Send the payment through to nodes[3] *without* clearing the PaymentClaimable event
 700         let mut send_events = nodes[0].node.get_and_clear_pending_msg_events();
 701         assert_eq!(send_events.len(), 2);
 702         let (node_1_msgs, mut send_events) = remove_first_msg_event_to_node(&nodes[1].node.get_our_node_id(), &send_events);
 703         let (node_2_msgs, _send_events) = remove_first_msg_event_to_node(&nodes[2].node.get_our_node_id(), &send_events);
 704         do_pass_along_path(&nodes[0], &[&nodes[1], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_1_msgs, true, false, None);
 705         do_pass_along_path(&nodes[0], &[&nodes[2], &nodes[3]], 15_000_000, payment_hash, Some(payment_secret), node_2_msgs, true, false, None);
 706
 707         // Now that we have an MPP payment pending, get the latest encoded copies of nodes[3]'s
 708         // monitors and ChannelManager, for use later, if we don't want to persist both monitors.
 709         let mut original_monitor = test_utils::TestVecWriter(Vec::new());
 710         if !persist_both_monitors {
 711                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 712                         if outpoint.to_channel_id() == chan_id_not_persisted {
 713                                 assert!(original_monitor.0.is_empty());
 714                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 715                         }
 716                 }
 717         }
 718
 719         let original_manager = nodes[3].node.encode();
 720
 721         expect_payment_claimable!(nodes[3], payment_hash, payment_secret, 15_000_000);
 722
 723         nodes[3].node.claim_funds(payment_preimage);
 724         check_added_monitors!(nodes[3], 2);
 725         expect_payment_claimed!(nodes[3], payment_hash, 15_000_000);
 726
 727         // Now fetch one of the two updated ChannelMonitors from nodes[3], and restart pretending we
 728         // crashed in between the two persistence calls - using one old ChannelMonitor and one new one,
 729         // with the old ChannelManager.
 730         let mut updated_monitor = test_utils::TestVecWriter(Vec::new());
 731         for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 732                 if outpoint.to_channel_id() == chan_id_persisted {
 733                         assert!(updated_monitor.0.is_empty());
 734                         nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut updated_monitor).unwrap();
 735                 }
 736         }
 737         // If `persist_both_monitors` is set, get the second monitor here as well
 738         if persist_both_monitors {
 739                 for outpoint in nodes[3].chain_monitor.chain_monitor.list_monitors() {
 740                         if outpoint.to_channel_id() == chan_id_not_persisted {
 741                                 assert!(original_monitor.0.is_empty());
 742                                 nodes[3].chain_monitor.chain_monitor.get_monitor(outpoint).unwrap().write(&mut original_monitor).unwrap();
 743                         }
 744                 }
 745         }
 746
 747         // Now restart nodes[3].
 748         reload_node!(nodes[3], original_manager, &[&updated_monitor.0, &original_monitor.0], persister, new_chain_monitor, nodes_3_deserialized);
 749
 750         // On startup the preimage should have been copied into the non-persisted monitor:
 751         assert!(get_monitor!(nodes[3], chan_id_persisted).get_stored_preimages().contains_key(&payment_hash));
 752         assert!(get_monitor!(nodes[3], chan_id_not_persisted).get_stored_preimages().contains_key(&payment_hash));
 753
 754         nodes[1].node.peer_disconnected(&nodes[3].node.get_our_node_id(), false);
 755         nodes[2].node.peer_disconnected(&nodes[3].node.get_our_node_id(), false);
 756
 757         // During deserialization, we should have closed one channel and broadcast its latest
 758         // commitment transaction. We should also still have the original PaymentClaimable event we
 759         // never finished processing.
 760         let events = nodes[3].node.get_and_clear_pending_events();
 761         assert_eq!(events.len(), if persist_both_monitors { 4 } else { 3 });
 762         if let Event::PaymentClaimable { amount_msat: 15_000_000, .. } = events[0] { } else { panic!(); }
 763         if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[1] { } else { panic!(); }
 764         if persist_both_monitors {
 765                 if let Event::ChannelClosed { reason: ClosureReason::OutdatedChannelManager, .. } = events[2] { } else { panic!(); }
 766         }
 767
 768         // On restart, we should also get a duplicate PaymentClaimed event as we persisted the
 769         // ChannelManager prior to handling the original one.
 770         if let Event::PaymentClaimed { payment_hash: our_payment_hash, amount_msat: 15_000_000, .. } =
 771                 events[if persist_both_monitors { 3 } else { 2 }]
 772         {
 773                 assert_eq!(payment_hash, our_payment_hash);
 774         } else { panic!(); }
 775
 776         assert_eq!(nodes[3].node.list_channels().len(), if persist_both_monitors { 0 } else { 1 });
 777         if !persist_both_monitors {
 778                 // If one of the two channels is still live, reveal the payment preimage over it.
 779
 780                 nodes[3].node.peer_connected(&nodes[2].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 781                 let reestablish_1 = get_chan_reestablish_msgs!(nodes[3], nodes[2]);
 782                 nodes[2].node.peer_connected(&nodes[3].node.get_our_node_id(), &msgs::Init { features: channelmanager::provided_init_features(), remote_network_address: None }).unwrap();
 783                 let reestablish_2 = get_chan_reestablish_msgs!(nodes[2], nodes[3]);
 784
 785                 nodes[2].node.handle_channel_reestablish(&nodes[3].node.get_our_node_id(), &reestablish_1[0]);
 786                 get_event_msg!(nodes[2], MessageSendEvent::SendChannelUpdate, nodes[3].node.get_our_node_id());
 787                 assert!(nodes[2].node.get_and_clear_pending_msg_events().is_empty());
 788
 789                 nodes[3].node.handle_channel_reestablish(&nodes[2].node.get_our_node_id(), &reestablish_2[0]);
 790
 791                 // Once we call `get_and_clear_pending_msg_events` the holding cell is cleared and the HTLC
 792                 // claim should fly.
 793                 let ds_msgs = nodes[3].node.get_and_clear_pending_msg_events();
 794                 check_added_monitors!(nodes[3], 1);
 795                 assert_eq!(ds_msgs.len(), 2);
 796                 if let MessageSendEvent::SendChannelUpdate { .. } = ds_msgs[0] {} else { panic!(); }
 797
 798                 let cs_updates = match ds_msgs[1] {
 799                         MessageSendEvent::UpdateHTLCs { ref updates, .. } => {
 800                                 nodes[2].node.handle_update_fulfill_htlc(&nodes[3].node.get_our_node_id(), &updates.update_fulfill_htlcs[0]);
 801                                 check_added_monitors!(nodes[2], 1);
 802                                 let cs_updates = get_htlc_update_msgs!(nodes[2], nodes[0].node.get_our_node_id());
 803                                 expect_payment_forwarded!(nodes[2], nodes[0], nodes[3], Some(1000), false, false);
 804                                 commitment_signed_dance!(nodes[2], nodes[3], updates.commitment_signed, false, true);
 805                                 cs_updates
 806                         }
 807                         _ => panic!(),
 808                 };
 809
 810                 nodes[0].node.handle_update_fulfill_htlc(&nodes[2].node.get_our_node_id(), &cs_updates.update_fulfill_htlcs[0]);
 811                 commitment_signed_dance!(nodes[0], nodes[2], cs_updates.commitment_signed, false, true);
 812                 expect_payment_sent!(nodes[0], payment_preimage);
 813         }
 814 }
 815
 816 #[test]
 817 fn test_partial_claim_before_restart() {
 818         do_test_partial_claim_before_restart(false);
 819         do_test_partial_claim_before_restart(true);
 820 }
 821
 822 fn do_forwarded_payment_no_manager_persistence(use_cs_commitment: bool, claim_htlc: bool, use_intercept: bool) {
 823         if !use_cs_commitment { assert!(!claim_htlc); }
 824         // If we go to forward a payment, and the ChannelMonitor persistence completes, but the
 825         // ChannelManager does not, we shouldn't try to forward the payment again, nor should we fail
 826         // it back until the ChannelMonitor decides the fate of the HTLC.
 827         // This was never an issue, but it may be easy to regress here going forward.
 828         let chanmon_cfgs = create_chanmon_cfgs(3);
 829         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 830         let mut intercept_forwards_config = test_default_channel_config();
 831         intercept_forwards_config.accept_intercept_htlcs = true;
 832         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, Some(intercept_forwards_config), None]);
 833
 834         let persister;
 835         let new_chain_monitor;
 836         let nodes_1_deserialized;
 837
 838         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
 839
 840         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 841         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
 842
 843         let intercept_scid = nodes[1].node.get_intercept_scid();
 844
 845         let (mut route, payment_hash, payment_preimage, payment_secret) =
 846                 get_route_and_payment_hash!(nodes[0], nodes[2], 1_000_000);
 847         if use_intercept {
 848                 route.paths[0][1].short_channel_id = intercept_scid;
 849         }
 850         let payment_id = PaymentId(nodes[0].keys_manager.backing.get_secure_random_bytes());
 851         let htlc_expiry = nodes[0].best_block_info().1 + TEST_FINAL_CLTV;
 852         nodes[0].node.send_payment(&route, payment_hash, &Some(payment_secret), payment_id).unwrap();
 853         check_added_monitors!(nodes[0], 1);
 854
 855         let payment_event = SendEvent::from_node(&nodes[0]);
 856         nodes[1].node.handle_update_add_htlc(&nodes[0].node.get_our_node_id(), &payment_event.msgs[0]);
 857         commitment_signed_dance!(nodes[1], nodes[0], payment_event.commitment_msg, false);
 858
 859         // Store the `ChannelManager` before handling the `PendingHTLCsForwardable`/`HTLCIntercepted`
 860         // events, expecting either event (and the HTLC itself) to be missing on reload even though its
 861         // present when we serialized.
 862         let node_encoded = nodes[1].node.encode();
 863
 864         let mut intercept_id = None;
 865         let mut expected_outbound_amount_msat = None;
 866         if use_intercept {
 867                 let events = nodes[1].node.get_and_clear_pending_events();
 868                 assert_eq!(events.len(), 1);
 869                 match events[0] {
 870                         Event::HTLCIntercepted { intercept_id: ev_id, expected_outbound_amount_msat: ev_amt, .. } => {
 871                                 intercept_id = Some(ev_id);
 872                                 expected_outbound_amount_msat = Some(ev_amt);
 873                         },
 874                         _ => panic!()
 875                 }
 876                 nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_2,
 877                         nodes[2].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap();
 878         }
 879
 880         expect_pending_htlcs_forwardable!(nodes[1]);
 881
 882         let payment_event = SendEvent::from_node(&nodes[1]);
 883         nodes[2].node.handle_update_add_htlc(&nodes[1].node.get_our_node_id(), &payment_event.msgs[0]);
 884         nodes[2].node.handle_commitment_signed(&nodes[1].node.get_our_node_id(), &payment_event.commitment_msg);
 885         check_added_monitors!(nodes[2], 1);
 886
 887         if claim_htlc {
 888                 get_monitor!(nodes[2], chan_id_2).provide_payment_preimage(&payment_hash, &payment_preimage,
 889                         &nodes[2].tx_broadcaster, &LowerBoundedFeeEstimator(nodes[2].fee_estimator), &nodes[2].logger);
 890         }
 891         assert!(nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().is_empty());
 892
 893         let _ = nodes[2].node.get_and_clear_pending_msg_events();
 894
 895         nodes[2].node.force_close_broadcasting_latest_txn(&chan_id_2, &nodes[1].node.get_our_node_id()).unwrap();
 896         let cs_commitment_tx = nodes[2].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 897         assert_eq!(cs_commitment_tx.len(), if claim_htlc { 2 } else { 1 });
 898
 899         check_added_monitors!(nodes[2], 1);
 900         check_closed_event!(nodes[2], 1, ClosureReason::HolderForceClosed);
 901         check_closed_broadcast!(nodes[2], true);
 902
 903         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
 904         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
 905         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
 906
 907         // Note that this checks that this is the only event on nodes[1], implying the
 908         // `HTLCIntercepted` event has been removed in the `use_intercept` case.
 909         check_closed_event!(nodes[1], 1, ClosureReason::OutdatedChannelManager);
 910
 911         if use_intercept {
 912                 // Attempt to forward the HTLC back out over nodes[1]' still-open channel, ensuring we get
 913                 // a intercept-doesn't-exist error.
 914                 let forward_err = nodes[1].node.forward_intercepted_htlc(intercept_id.unwrap(), &chan_id_1,
 915                         nodes[0].node.get_our_node_id(), expected_outbound_amount_msat.unwrap()).unwrap_err();
 916                 assert_eq!(forward_err, APIError::APIMisuseError {
 917                         err: format!("Payment with intercept id {} not found", log_bytes!(intercept_id.unwrap().0))
 918                 });
 919         }
 920
 921         let bs_commitment_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 922         assert_eq!(bs_commitment_tx.len(), 1);
 923
 924         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
 925         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
 926
 927         if use_cs_commitment {
 928                 // If we confirm a commitment transaction that has the HTLC on-chain, nodes[1] should wait
 929                 // for an HTLC-spending transaction before it does anything with the HTLC upstream.
 930                 confirm_transaction(&nodes[1], &cs_commitment_tx[0]);
 931                 assert!(nodes[1].node.get_and_clear_pending_events().is_empty());
 932                 assert!(nodes[1].node.get_and_clear_pending_msg_events().is_empty());
 933
 934                 if claim_htlc {
 935                         confirm_transaction(&nodes[1], &cs_commitment_tx[1]);
 936                 } else {
 937                         connect_blocks(&nodes[1], htlc_expiry - nodes[1].best_block_info().1);
 938                         let bs_htlc_timeout_tx = nodes[1].tx_broadcaster.txn_broadcasted.lock().unwrap().split_off(0);
 939                         assert_eq!(bs_htlc_timeout_tx.len(), 1);
 940                         confirm_transaction(&nodes[1], &bs_htlc_timeout_tx[0]);
 941                 }
 942         } else {
 943                 confirm_transaction(&nodes[1], &bs_commitment_tx[0]);
 944         }
 945
 946         if !claim_htlc {
 947                 expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
 948         } else {
 949                 expect_payment_forwarded!(nodes[1], nodes[0], nodes[2], Some(1000), false, true);
 950         }
 951         check_added_monitors!(nodes[1], 1);
 952
 953         let events = nodes[1].node.get_and_clear_pending_msg_events();
 954         assert_eq!(events.len(), 1);
 955         match &events[0] {
 956                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fulfill_htlcs, update_fail_htlcs, commitment_signed, .. }, .. } => {
 957                         if claim_htlc {
 958                                 nodes[0].node.handle_update_fulfill_htlc(&nodes[1].node.get_our_node_id(), &update_fulfill_htlcs[0]);
 959                         } else {
 960                                 nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
 961                         }
 962                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
 963                 },
 964                 _ => panic!("Unexpected event"),
 965         }
 966
 967         if claim_htlc {
 968                 expect_payment_sent!(nodes[0], payment_preimage);
 969         } else {
 970                 expect_payment_failed!(nodes[0], payment_hash, false);
 971         }
 972 }
 973
 974 #[test]
 975 fn forwarded_payment_no_manager_persistence() {
 976         do_forwarded_payment_no_manager_persistence(true, true, false);
 977         do_forwarded_payment_no_manager_persistence(true, false, false);
 978         do_forwarded_payment_no_manager_persistence(false, false, false);
 979 }
 980
 981 #[test]
 982 fn intercepted_payment_no_manager_persistence() {
 983         do_forwarded_payment_no_manager_persistence(true, true, true);
 984         do_forwarded_payment_no_manager_persistence(true, false, true);
 985         do_forwarded_payment_no_manager_persistence(false, false, true);
 986 }
 987
 988 #[test]
 989 fn removed_payment_no_manager_persistence() {
 990         // If an HTLC is failed to us on a channel, and the ChannelMonitor persistence completes, but
 991         // the corresponding ChannelManager persistence does not, we need to ensure that the HTLC is
 992         // still failed back to the previous hop even though the ChannelMonitor now no longer is aware
 993         // of the HTLC. This was previously broken as no attempt was made to figure out which HTLCs
 994         // were left dangling when a channel was force-closed due to a stale ChannelManager.
 995         let chanmon_cfgs = create_chanmon_cfgs(3);
 996         let node_cfgs = create_node_cfgs(3, &chanmon_cfgs);
 997         let node_chanmgrs = create_node_chanmgrs(3, &node_cfgs, &[None, None, None]);
 998
 999         let persister;
1000         let new_chain_monitor;
1001         let nodes_1_deserialized;
1002
1003         let mut nodes = create_network(3, &node_cfgs, &node_chanmgrs);
1004
1005         let chan_id_1 = create_announced_chan_between_nodes(&nodes, 0, 1, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
1006         let chan_id_2 = create_announced_chan_between_nodes(&nodes, 1, 2, channelmanager::provided_init_features(), channelmanager::provided_init_features()).2;
1007
1008         let (_, payment_hash, _) = route_payment(&nodes[0], &[&nodes[1], &nodes[2]], 1_000_000);
1009
1010         let node_encoded = nodes[1].node.encode();
1011
1012         nodes[2].node.fail_htlc_backwards(&payment_hash);
1013         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[2], [HTLCDestination::FailedPayment { payment_hash }]);
1014         check_added_monitors!(nodes[2], 1);
1015         let events = nodes[2].node.get_and_clear_pending_msg_events();
1016         assert_eq!(events.len(), 1);
1017         match &events[0] {
1018                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1019                         nodes[1].node.handle_update_fail_htlc(&nodes[2].node.get_our_node_id(), &update_fail_htlcs[0]);
1020                         commitment_signed_dance!(nodes[1], nodes[2], commitment_signed, false);
1021                 },
1022                 _ => panic!("Unexpected event"),
1023         }
1024
1025         let chan_0_monitor_serialized = get_monitor!(nodes[1], chan_id_1).encode();
1026         let chan_1_monitor_serialized = get_monitor!(nodes[1], chan_id_2).encode();
1027         reload_node!(nodes[1], node_encoded, &[&chan_0_monitor_serialized, &chan_1_monitor_serialized], persister, new_chain_monitor, nodes_1_deserialized);
1028
1029         match nodes[1].node.pop_pending_event().unwrap() {
1030                 Event::ChannelClosed { ref reason, .. } => {
1031                         assert_eq!(*reason, ClosureReason::OutdatedChannelManager);
1032                 },
1033                 _ => panic!("Unexpected event"),
1034         }
1035
1036         // Now that the ChannelManager has force-closed the channel which had the HTLC removed, it is
1037         // now forgotten everywhere. The ChannelManager should have, as a side-effect of reload,
1038         // learned that the HTLC is gone from the ChannelMonitor and added it to the to-fail-back set.
1039         nodes[0].node.peer_disconnected(&nodes[1].node.get_our_node_id(), true);
1040         reconnect_nodes(&nodes[0], &nodes[1], (false, false), (0, 0), (0, 0), (0, 0), (0, 0), (0, 0), (false, false));
1041
1042         expect_pending_htlcs_forwardable_and_htlc_handling_failed!(nodes[1], [HTLCDestination::NextHopChannel { node_id: Some(nodes[2].node.get_our_node_id()), channel_id: chan_id_2 }]);
1043         check_added_monitors!(nodes[1], 1);
1044         let events = nodes[1].node.get_and_clear_pending_msg_events();
1045         assert_eq!(events.len(), 1);
1046         match &events[0] {
1047                 MessageSendEvent::UpdateHTLCs { updates: msgs::CommitmentUpdate { update_fail_htlcs, commitment_signed, .. }, .. } => {
1048                         nodes[0].node.handle_update_fail_htlc(&nodes[1].node.get_our_node_id(), &update_fail_htlcs[0]);
1049                         commitment_signed_dance!(nodes[0], nodes[1], commitment_signed, false);
1050                 },
1051                 _ => panic!("Unexpected event"),
1052         }
1053
1054         expect_payment_failed!(nodes[0], payment_hash, false);
1055 }